diff --git a/CMakeLists.txt b/CMakeLists.txt index f4f2fd98f..1b1fe799d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,129 @@ option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON) option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF) option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) +# Initialize bundle +# ----------------- +ecbuild_bundle_initialize() + +# Build bundle source code. +if(BUILD_GDASBUNDLE) + +# jedi-cmake + ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda/jedi-cmake.git" TAG 1.4.0 RECURSIVE ) + include( jedicmake/cmake/Functions/git_functions.cmake ) + +# ECMWF libraries + option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF + option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF + option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF + +# turn off optional OOPS toy models + option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) + option( ENABLE_QG_MODEL "Build QG toy model" OFF ) + + ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) + ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) + ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) + +# External (required) observation operators + option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF + ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) + +# Build GSI-B + option(BUILD_GSIBEC "Build GSI-B" OFF) + if(BUILD_GSIBEC) + ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" TAG 1.1.2 ) + endif() + +# Gibbs seawater + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop ) + +# Core JEDI repositories + ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" BRANCH develop) + ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop ) + ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop ) + option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop ) + option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) + # ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA-internal/ufo.git" BRANCH feature/crtm ) +# FMS and FV3 dynamical core + ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) + ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable ) + +# fv3-jedi and associated repositories + ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop ) + ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop ) + option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop ) + +# SOCA associated repositories +# TODO: Move the Icepack fork to EMC github + set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") + if ( BUILD_ICEPACK ) + ecbuild_bundle( PROJECT icepack GIT "https://github.com/JCSDA-internal/Icepack.git" BRANCH feature/ecbuild-new ) + endif() + ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" BRANCH main-ecbuild RECURSIVE ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop ) + + # Build JEDI/DA or other peripherals + ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) + +# Build IODA converters + option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) + if(BUILD_IODA_CONVERTERS) + ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop ) + endif() + +# Land associated repositories + ecbuild_bundle( PROJECT land-imsproc GIT "https://github.com/NOAA-PSL/land-IMS_proc.git" TAG 6373819 ) + ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git" TAG 2923344) + +# ioda, ufo, fv3-jedi, and saber test data +#--------------------------------- +if(CLONE_JCSDADATA) + + # If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch. + # If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case, + # IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned. + # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored + # in a local directory, ioda-data repo will not be cloned + + find_branch_name(REPO_DIR_NAME ioda) + # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored + # in a local directory, ioda-data repo will not be cloned + if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) + ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop ) + + # If IODA's current branch is available in ioda-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME ioda-data + BRANCH ${GIT_BRANCH_FUNC} ) + endif() + + # same procedure for ufo-data + find_branch_name(REPO_DIR_NAME ufo) + if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) + ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop ) + + # If UFO's current branch is available in ioda-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME ufo-data + BRANCH ${GIT_BRANCH_FUNC} ) + endif() + + # same procedure for fv3-jedi-data + find_branch_name(REPO_DIR_NAME fv3-jedi) + if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop ) + + # If fv3-jedi's current branch is available in ioda-data repo, that branch will be checked out + branch_checkout (REPO_DIR_NAME fv3-jedi-data + BRANCH ${GIT_BRANCH_FUNC} ) + endif() + + endif(CLONE_JCSDADATA) + +endif(BUILD_GDASBUNDLE) + # Install utility scripts. add_subdirectory(ush) diff --git a/parm/atm/obs/config/amsua_metop-a.yaml b/parm/atm/obs/config/amsua_metop-a.yaml new file mode 100644 index 000000000..8b7da213f --- /dev/null +++ b/parm/atm/obs/config/amsua_metop-a.yaml @@ -0,0 +1,524 @@ +obs space: + name: amsua_metop-a + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-a.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_metop-a_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-a_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-a + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-a.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-a.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-a_tlapse $(DATA)/obs/$(GPREFIX)amsua_metop-a.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-a.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-a.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate Derived Variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_metop-a_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_metop-a_channels + - name: HofX/brightnessTemperature + channels: *amsua_metop-a_channels + coefs: [1, -1] + +# Step 0-C: Assign Initial All-Sky Observation Error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window Channel Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation Error Inflation based on Topography Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-a_channels + +# Step 7: Obs Error Inflation based on TOA Transmittancec Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-a_channels + +# Step 8: Observation Error Inflation based on Surface Jacobian Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-a_channels + +# Step 9: Situation Dependent Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-a_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-a_channels + +# Step 10: Gross Check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_metop-a_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-a_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-a_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-a_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-Channel Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_metop-b.yaml b/parm/atm/obs/config/amsua_metop-b.yaml new file mode 100644 index 000000000..035a36c1b --- /dev/null +++ b/parm/atm/obs/config/amsua_metop-b.yaml @@ -0,0 +1,524 @@ +obs space: + name: amsua_metop-b + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-b.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_metop-b_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-b_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-b + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-b.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-b.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-b_tlapse $(DATA)/obs/$(GPREFIX)amsua_metop-b.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-b_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-b.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-b.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate Derived Variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_metop-b_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_metop-b_channels + - name: HofX/brightnessTemperature + channels: *amsua_metop-b_channels + coefs: [1, -1] + +# Step 0-C: Assign Initial All-Sky Observation Error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window Channel Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation Error Inflation based on Topography Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-b_channels + +# Step 7: Obs Error Inflation based on TOA Transmittancec Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-b_channels + +# Step 8: Observation Error Inflation based on Surface Jacobian Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-b_channels + +# Step 9: Situation Dependent Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-b_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-b_channels + +# Step 10: Gross Check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_metop-b_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-b_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-b_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-b_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-Channel Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_metop-c.yaml b/parm/atm/obs/config/amsua_metop-c.yaml new file mode 100644 index 000000000..3dceb20ed --- /dev/null +++ b/parm/atm/obs/config/amsua_metop-c.yaml @@ -0,0 +1,524 @@ +obs space: + name: amsua_metop-c + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_metop-c.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_metop-c_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-c_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-c + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-c.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-c.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-c_tlapse $(DATA)/obs/$(GPREFIX)amsua_metop-c.tlapse.txt + - name: lapse_rate + tlapse: *amsua_metop-c_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_metop-c.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_metop-c.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate Derived Variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_metop-c_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_metop-c_channels + - name: HofX/brightnessTemperature + channels: *amsua_metop-c_channels + coefs: [1, -1] + +# Step 0-C: Assign Initial All-Sky Observation Error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window Channel Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation Error Inflation based on Topography Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-c_channels + +# Step 7: Obs Error Inflation based on TOA Transmittancec Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-c_channels + +# Step 8: Observation Error Inflation based on Surface Jacobian Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_metop-c_channels + +# Step 9: Situation Dependent Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_metop-c_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_metop-c_channels + +# Step 10: Gross Check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_metop-c_channels + options: + sensor: *Sensor_ID + channels: *amsua_metop-c_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_metop-c_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_metop-c_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-Channel Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n15.yaml b/parm/atm/obs/config/amsua_n15.yaml new file mode 100644 index 000000000..2d191a38a --- /dev/null +++ b/parm/atm/obs/config/amsua_n15.yaml @@ -0,0 +1,524 @@ +obs space: + name: amsua_n15 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n15.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_n15_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_n15_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n15 + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_n15.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n15.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n15_tlapse $(DATA)/obs/$(GPREFIX)amsua_n15.tlapse.txt + - name: lapse_rate + tlapse: *amsua_n15_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_n15.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_n15.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate Derived Variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_n15_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_n15_channels + - name: HofX/brightnessTemperature + channels: *amsua_n15_channels + coefs: [1, -1] + +# Step 0-C: Assign Initial All-Sky Observation Error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window Channel Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation Error Inflation based on Topography Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n15_channels + +# Step 7: Obs Error Inflation based on TOA Transmittancec Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n15_channels + +# Step 8: Observation Error Inflation based on Surface Jacobian Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n15_channels + +# Step 9: Situation Dependent Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n15_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n15_channels + +# Step 10: Gross Check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_n15_channels + options: + sensor: *Sensor_ID + channels: *amsua_n15_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n15_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n15_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-Channel Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n18.yaml b/parm/atm/obs/config/amsua_n18.yaml new file mode 100644 index 000000000..c5673e062 --- /dev/null +++ b/parm/atm/obs/config/amsua_n18.yaml @@ -0,0 +1,524 @@ +obs space: + name: amsua_n18 + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)amsua_n18.tm00.nc + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_amsua_n18_{{ current_cycle | to_YMDH }}.nc + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &amsua_n18_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: $(DATA)/crtm/ + +obs bias: + input file: $(DATA)/obs/$(GPREFIX)amsua_n18.satbias.nc4 + output file: $(DATA)/bc/$(APREFIX)amsua_n18.satbias.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n18_tlapse $(DATA)/obs/$(GPREFIX)amsua_n18.tlapse.txt + - name: lapse_rate + tlapse: *amsua_n18_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: $(DATA)/obs/$(GPREFIX)amsua_n18.satbias_cov.nc4 + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: $(DATA)/bc/$(APREFIX)amsua_n18.satbias_cov.nc4 + +obs pre filters: +# Step 0-A: Create Diagnostic Flags +- filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + +obs post filters: +# Step 0-B: Calculate Derived Variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate scattering index from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + +# Calculate CLW obs/bkg match index +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Calculate Innovation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_n18_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_n18_channels + - name: HofX/brightnessTemperature + channels: *amsua_n18_channels + coefs: [1, -1] + +# Step 0-C: Assign Initial All-Sky Observation Error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + +# Step 1: Remove Observations from the Edge of the Scan +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# Step 2: Data Thinning +- filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic +# round_horizontal_bin_count_to_nearest: true +# partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + +# Step 3A: CLW Retrieval Check (observation_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromObs + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 3B: CLW Retrieval Check (background_based) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/CLWRetFromBkg + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + +# Step 4: Window Channel Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + +# Step 5: Hydrometeor Check (cloud/precipitation affected chanels) +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUA + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + clwret_function: + name: DerivedMetaData/CLWRetFromObs + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + +# Step 6: Observation Error Inflation based on Topography Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n18_channels + +# Step 7: Obs Error Inflation based on TOA Transmittancec Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n18_channels + +# Step 8: Observation Error Inflation based on Surface Jacobian Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n18_channels + +# Step 9: Situation Dependent Check +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorFactorSituDependMW + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + scatobs_function: + name: DerivedMetaData/SIRetFromObs + clwmatchidx_function: + name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n18_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n18_channels + +# Step 10: Gross Check +# Remove data if abs(Obs-HofX) > absolute threhold +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorBoundMW + channels: *amsua_n18_channels + options: + sensor: *Sensor_ID + channels: *amsua_n18_channels + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n18_channels + obserr_function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n18_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + +# Step 11: Inter-Channel Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, 1, 1 ] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/config/amsua_n19.yaml.j2 b/parm/atm/obs/config/amsua_n19.yaml.j2 index 06ec22687..25f0a914a 100644 --- a/parm/atm/obs/config/amsua_n19.yaml.j2 +++ b/parm/atm/obs/config/amsua_n19.yaml.j2 @@ -3,7 +3,7 @@ obsdatain: engine: type: H5File - obsfile: '{{ DATA }}/obs/{{ OPREFIX }}amsua_n19.{{ current_cycle | to_YMDH }}.nc' + obsfile: '{{ DATA }}/obs/{{ OPREFIX }}amsua_n19.tm00.nc' obsdataout: engine: type: H5File @@ -12,15 +12,18 @@ max pool size: 1 simulated variables: [brightnessTemperature] channels: &amsua_n19_channels 1-15 + obs operator: name: CRTM Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: - Sensor_ID: amsua_n19 + Sensor_ID: &Sensor_ID amsua_n19 EndianType: little_endian CoefficientPath: '{{ DATA }}/crtm/' + obs bias: input file: '{{ DATA }}/obs/{{ GPREFIX }}amsua_n19.satbias.nc' output file: '{{ DATA }}/bc/{{ APREFIX }}amsua_n19.satbias.nc' @@ -51,24 +54,180 @@ ratio: 1.1 ratio for small dataset: 2.0 output file: '{{ DATA }}/bc/{{ APREFIX }}amsua_n19.satbias_cov.nc' - obs filters: - - filter: BlackList + + obs pre filters: + - filter: Create Diagnostic Flags filter variables: - name: brightnessTemperature channels: *amsua_n19_channels - action: - name: assign error - error function: + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + + obs post filters: + # Step 0-B: Calculate Derived Variables + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceParam + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 0 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 100 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 85 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 90 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 85 + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/thinningCriteria + type: int + function: + name: IntObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/surfaceParam + coefs: [1] + + # Calculate CLW retrieved from background + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + + # Calculate symmetric retrieved CLW + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + + - filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + + # Calculate scattering index from observation + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/SIRetFromObs + type: float + function: + name: ObsFunction/SCATRetMW + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 15 + scatret_types: [ObsValue] + + # Calculate CLW obs/bkg match index + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWMatchIndex + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/CLWMatchIndexMW + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + clwobs_function: + name: DerivedMetaData/CLWRetFromObs + clwbkg_function: + name: DerivedMetaData/CLWRetFromBkg + clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + + # Calculate symmetric observation error + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorModelRamp channels: *amsua_n19_channels options: channels: *amsua_n19_channels xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] + name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, 0.100, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.030] @@ -77,39 +236,109 @@ 0.000, 0.000, 0.000, 0.000, 0.200] err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] + 0.400, 0.550, 0.800, 4.000, 3.500] err1: [20.000, 18.000, 12.000, 3.000, 0.500, 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] - # CLW Retrieval Check + 0.400, 0.550, 0.800, 4.000, 18.000] + + # Calculate Innovation + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *amsua_n19_channels + options: + variables: + - name: ObsValue/brightnessTemperature + channels: *amsua_n19_channels + - name: HofX/brightnessTemperature + channels: *amsua_n19_channels + coefs: [1, -1] + + # Step 0-C: Assign Initial All-Sky Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + + # Step 1: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-15 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 4-27 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + + # Step 2: Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + priority_variable: DerivedMetaData/thinningCriteria + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + + # Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + - name: DerivedMetaData/CLWRetFromObs maxvalue: 999.0 - action: - name: reject - # CLW Retrieval Check + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 3B: CLW Retrieval Check (background_based) - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] + - name: DerivedMetaData/CLWRetFromBkg maxvalue: 999.0 - action: - name: reject - # Hydrometeor Check (cloud/precipitation affected chanels) + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 4: Window Channel Sanity Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + test variables: + - name: DerivedMetaData/Innovation + channels: 1, 2, 4-6, 15 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + + # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -121,217 +350,181 @@ channels: *amsua_n19_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] + 0.400, 0.550, 0.800, 4.000, 3.500] clwret_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] + name: DerivedMetaData/CLWRetFromObs obserr_function: - name: ObsFunction/ObsErrorModelRamp + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] maxvalue: 0.0 - action: - name: reject - # Topography check - - filter: BlackList - filter variables: - - name: brightnessTemperature + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + + # Step 6: Observation Error Inflation based on Topography Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels - action: - name: inflate error - inflation variable: + type: float + function: name: ObsFunction/ObsErrorFactorTopoRad channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels - # Transmittnace Top Check - - filter: BlackList + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorTopo + channels: *amsua_n19_channels + + # Step 7: Obs Error Inflation based on TOA Transmittancec Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorTransmitTopRad channels: *amsua_n19_channels options: - sensor: amsua_n19 channels: *amsua_n19_channels - # Surface Jacobian check - - filter: BlackList + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorTransmitTop + channels: *amsua_n19_channels + + # Step 8: Observation Error Inflation based on Surface Jacobian Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorSurfJacobianRad channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - # Situation dependent Check - - filter: BlackList + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels action: name: inflate error inflation variable: + name: DerivedMetaData/ObsErrorFactorSurfJacobian + channels: *amsua_n19_channels + + # Step 9: Situation Dependent Check + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n19_channels + type: float + function: name: ObsFunction/ObsErrorFactorSituDependMW channels: *amsua_n19_channels options: - sensor: amsua_n19 + sensor: *Sensor_ID channels: *amsua_n19_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - bias_application: HofX + name: DerivedMetaData/CLWRetFromBkg + clwobs_function: + name: DerivedMetaData/CLWRetFromObs scatobs_function: - name: ObsFunction/SCATRetMW - options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 15 - scatret_types: [ObsValue] - bias_application: HofX + name: DerivedMetaData/SIRetFromObs clwmatchidx_function: - name: ObsFunction/CLWMatchIndexMW + name: DerivedMetaData/CLWMatchIndex channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - bias_application: HofX - clwret_clearsky: [0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] obserr_function: - name: ObsFunction/ObsErrorModelRamp + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - options: - channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - # Gross check - - filter: Background Check + 0.400, 0.550, 0.800, 4.000, 3.500] + + - filter: Perform Action filter variables: - name: brightnessTemperature channels: *amsua_n19_channels - function absolute threshold: - - name: ObsFunction/ObsErrorBoundMW + action: + name: inflate error + inflation variable: + name: DerivedMetaData/ObsErrorFactorSituDepend + channels: *amsua_n19_channels + + # Step 10: Gross Check + # Remove data if abs(Obs-HofX) > absolute threhold + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorFactorLat + type: float + function: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBound channels: *amsua_n19_channels - options: - sensor: amsua_n19 + type: float + function: + name: ObsFunction/ObsErrorBoundMW channels: *amsua_n19_channels - obserr_bound_latitude: - name: ObsFunction/ObsErrorFactorLatRad - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsFunction/ObsErrorFactorTransmitTopRad + options: + sensor: *Sensor_ID channels: *amsua_n19_channels - options: + obserr_bound_latitude: + name: DerivedMetaData/ObsErrorFactorLat + obserr_bound_transmittop: + name: DerivedMetaData/ObsErrorFactorTransmitTop channels: *amsua_n19_channels - obserr_bound_topo: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *amsua_n19_channels - options: + options: + channels: *amsua_n19_channels + obserr_bound_topo: + name: DerivedMetaData/ObsErrorFactorTopo channels: *amsua_n19_channels - sensor: amsua_n19 - obserr_function: - name: ObsFunction/ObsErrorModelRamp - channels: *amsua_n19_channels - options: + obserr_function: + name: DerivedMetaData/InitialObsError channels: *amsua_n19_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - bias_application: HofX - x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, - 0.100, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.030] - x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, - 1.500, 0.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.200] - err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 3.500] - err1: [20.000, 18.000, 12.000, 3.000, 0.500, - 0.300, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 3.000, 18.000] - obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, - 2.5, 3.5, 4.5, 4.5, 4.5] - action: - name: reject - # Inter-channel check + threhold: 3 + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + function absolute threshold: + - name: DerivedMetaData/ObsErrorBound + channels: *amsua_n19_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + + # Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -341,14 +534,19 @@ channels: *amsua_n19_channels options: channels: *amsua_n19_channels - sensor: amsua_n19 + use passive_bc: true + sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, - 1, 1, 1, -1, 1 ] + 1, 1, 1, 1, 1 ] maxvalue: 1.0e-12 - action: - name: reject - # Useflag check + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + + # Step 12: Useflag Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -361,7 +559,10 @@ channels: *amsua_n19_channels use_flag: [ 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, - 1, 1, 1, -1, 1 ] + 1, 1, 1, 1, 1 ] minvalue: 1.0e-12 - action: - name: reject + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject diff --git a/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 b/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 index a8bc25019..8e9c7ba7c 100644 --- a/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 +++ b/parm/atm/obs/lists/gdas_prototype_3d.yaml.j2 @@ -1,6 +1,7 @@ observers: {% filter indent(width=2) %} {% include 'atm/obs/config/satwind_abi_goes-16.yaml.j2' %} +{% include 'atm/obs/config/amsua_n19.yaml.j2' %} {% include 'atm/obs/config/scatwind_ascat_metop-b.yaml.j2' %} {% include 'atm/obs/config/conv_ps.yaml.j2' %} {% include 'atm/obs/config/gnssro.yaml.j2' %} diff --git a/parm/atm/obs/testing/amsua_metop-a.yaml b/parm/atm/obs/testing/amsua_metop-a.yaml index d2951e032..2f6dc5696 100644 --- a/parm/atm/obs/testing/amsua_metop-a.yaml +++ b/parm/atm/obs/testing/amsua_metop-a.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-a - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-a obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV amsua_metop-a_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_metop-a_channels 1-15 + geovals: filename: !ENV amsua_metop-a_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-a + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-a_satbias_${GDATE}.nc4 variational bc: @@ -40,49 +43,40 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +obs post filters: +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -153,13 +147,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +166,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +192,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_metop-a_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -232,7 +226,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +239,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,17 +253,21 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,96 +275,96 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-a_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-a_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-a_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +373,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-a_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,14 +384,14 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-a_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -407,27 +405,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-a_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_metop-a_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-a_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-a_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-a_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,26 +434,26 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-a_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-a_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_metop-a_channels options: - channels: *all_channels + channels: *amsua_metop-a_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -464,8 +462,30 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 93421 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 91978 +#passedBenchmark: 91978 # no useflag check diff --git a/parm/atm/obs/testing/amsua_metop-a_noqc.yaml b/parm/atm/obs/testing/amsua_metop-a_noqc.yaml new file mode 100644 index 000000000..f029fd8ab --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-a_noqc.yaml @@ -0,0 +1,141 @@ +obs space: + name: amsua_metop-a + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-a_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-a_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-a_channels 1-15 + +geovals: + filename: !ENV amsua_metop-a_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-a + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV amsua_metop-a_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-a_tlap !ENV amsua_metop-a_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-a_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-a_channels + options: + channels: *amsua_metop-a_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-a_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-a_channels + +passedBenchmark: 117872 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 + diff --git a/parm/atm/obs/testing/amsua_metop-b.yaml b/parm/atm/obs/testing/amsua_metop-b.yaml index 3c666d8e9..0890c213a 100644 --- a/parm/atm/obs/testing/amsua_metop-b.yaml +++ b/parm/atm/obs/testing/amsua_metop-b.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-b - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-b obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV amsua_metop-b_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_metop-b_channels 1-15 + geovals: filename: !ENV amsua_metop-b_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-b + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-b_satbias_${GDATE}.nc4 variational bc: @@ -40,49 +43,40 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +obs post filters: +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -153,13 +147,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +166,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +192,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_metop-b_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -232,7 +226,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +239,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,17 +253,21 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,96 +275,96 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-b_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-b_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-b_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +373,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-b_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,14 +384,14 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-b_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -407,27 +405,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-b_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_metop-b_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-b_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-b_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-b_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,36 +434,58 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-b_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-b_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_metop-b_channels options: - channels: *all_channels + channels: *amsua_metop-b_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [ -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, - 1, 1, 1, 1, -1 ] + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 71068 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + use_flag: [ -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, + 1, 1, 1, 1, -1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 62069 +#passedBenchmark: 70974 # no useflag check diff --git a/parm/atm/obs/testing/amsua_metop-b_noqc.yaml b/parm/atm/obs/testing/amsua_metop-b_noqc.yaml new file mode 100644 index 000000000..4b5aa7cbb --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-b_noqc.yaml @@ -0,0 +1,140 @@ +obs space: + name: amsua_metop-b + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-b_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-b_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-b_channels 1-15 + +geovals: + filename: !ENV amsua_metop-b_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-b + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV amsua_metop-b_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-b_tlap !ENV amsua_metop-b_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-b_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-b_channels + options: + channels: *amsua_metop-b_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-b_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-b_channels + +passedBenchmark: 125148 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_metop-c.yaml b/parm/atm/obs/testing/amsua_metop-c.yaml index 0e9e82795..cc8bb7330 100644 --- a/parm/atm/obs/testing/amsua_metop-c.yaml +++ b/parm/atm/obs/testing/amsua_metop-c.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_metop-c - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_metop-c obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV amsua_metop-c_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_metop-c_channels 1-15 + geovals: filename: !ENV amsua_metop-c_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-c + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_metop-c_satbias_${GDATE}.nc4 variational bc: @@ -40,49 +43,40 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +obs post filters: +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -153,13 +147,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +166,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +192,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_metop-c_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -232,7 +226,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +239,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,17 +253,21 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,96 +275,96 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-c_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-c_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_metop-c_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +373,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_metop-c_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,14 +384,14 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_metop-c_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -407,27 +405,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-c_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_metop-c_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_metop-c_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_metop-c_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_metop-c_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,26 +434,26 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_metop-c_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_metop-c_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_metop-c_channels options: - channels: *all_channels + channels: *amsua_metop-c_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -464,7 +462,29 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck ignore: rejected observations - name: reject -passedBenchmark: 113345 + +passedBenchmark: 112377 +#passedBenchmark: 112377 # no useflag check diff --git a/parm/atm/obs/testing/amsua_metop-c_noqc.yaml b/parm/atm/obs/testing/amsua_metop-c_noqc.yaml new file mode 100644 index 000000000..9624b230c --- /dev/null +++ b/parm/atm/obs/testing/amsua_metop-c_noqc.yaml @@ -0,0 +1,140 @@ +obs space: + name: amsua_metop-c + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_metop-c_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_metop-c_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_metop-c_channels 1-15 + +geovals: + filename: !ENV amsua_metop-c_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_metop-c + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV amsua_metop-c_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_metop-c_tlap !ENV amsua_metop-c_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_metop-c_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_metop-c_channels + options: + channels: *amsua_metop-c_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_metop-c_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_metop-c_channels + +passedBenchmark: 133601 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_n15.yaml b/parm/atm/obs/testing/amsua_n15.yaml index 973a2f26a..3bd1c0932 100644 --- a/parm/atm/obs/testing/amsua_n15.yaml +++ b/parm/atm/obs/testing/amsua_n15.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n15 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n15 obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV amsua_n15_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_n15_channels 1-15 + geovals: filename: !ENV amsua_n15_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n15 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n15_satbias_${GDATE}.nc4 variational bc: @@ -40,49 +43,40 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +obs post filters: +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -153,13 +147,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +166,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +192,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_n15_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -232,7 +226,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +239,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,17 +253,21 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, 0.230, 0.250, 0.275, 0.340, 0.400, 0.600, 1.000, 1.500, 4.000, 3.500] @@ -277,96 +275,96 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n15_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n15_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n15_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +373,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n15_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels obserr_clearsky: [ 3.000, 2.200, 2.000, 0.600, 0.300, 0.230, 0.250, 0.275, 0.340, 0.400, 0.600, 1.000, 1.500, 4.000, 3.500] @@ -386,14 +384,14 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n15_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -407,27 +405,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n15_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_n15_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n15_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n15_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n15_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,26 +434,26 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n15_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n15_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_n15_channels options: - channels: *all_channels + channels: *amsua_n15_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -464,7 +462,29 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 91792 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + use_flag: [ 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + -1, 1, 1, -1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 80770 +#passedBenchmark: 85704 # no useflag check diff --git a/parm/atm/obs/testing/amsua_n15_noqc.yaml b/parm/atm/obs/testing/amsua_n15_noqc.yaml new file mode 100644 index 000000000..c94d3ddb2 --- /dev/null +++ b/parm/atm/obs/testing/amsua_n15_noqc.yaml @@ -0,0 +1,140 @@ +obs space: + name: amsua_n15 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n15_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n15_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_n15_channels 1-15 + +geovals: + filename: !ENV amsua_n15_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n15 + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV amsua_n15_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n15_tlap !ENV amsua_n15_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n15_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n15_channels + options: + channels: *amsua_n15_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 3.000, 2.200, 2.000, 0.600, 0.300, + 0.230, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.250, 0.275, 0.340, 0.400, + 0.600, 1.000, 1.500, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n15_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n15_channels + +passedBenchmark: 116256 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_n18.yaml b/parm/atm/obs/testing/amsua_n18.yaml index d11eba99b..33206b14d 100644 --- a/parm/atm/obs/testing/amsua_n18.yaml +++ b/parm/atm/obs/testing/amsua_n18.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n18 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n18 obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV amsua_n18_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_n18_channels 1-15 + geovals: filename: !ENV amsua_n18_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n18_satbias_${GDATE}.nc4 variational bc: @@ -40,49 +43,40 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +obs post filters: +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -153,13 +147,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +166,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +192,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_n18_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -232,7 +226,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,10 +239,10 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -259,17 +253,21 @@ obs post filters: maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,96 +275,96 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n18_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n18_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n18_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +373,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n18_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,14 +384,14 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n18_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -407,27 +405,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n18_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_n18_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n18_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n18_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n18_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,26 +434,26 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n18_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n18_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_n18_channels options: - channels: *all_channels + channels: *amsua_n18_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, -1, @@ -464,8 +462,30 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 116999 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + use_flag: [ 1, 1, 1, 1, -1, + 1, 1, -1, -1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 92176 +#passedBenchmark: 115991 # no useflag check diff --git a/parm/atm/obs/testing/amsua_n18_noqc.yaml b/parm/atm/obs/testing/amsua_n18_noqc.yaml new file mode 100644 index 000000000..597ab563d --- /dev/null +++ b/parm/atm/obs/testing/amsua_n18_noqc.yaml @@ -0,0 +1,140 @@ +obs space: + name: amsua_n18 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n18_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n18_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_n18_channels 1-15 + +geovals: + filename: !ENV amsua_n18_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n18 + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV amsua_n18_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n18_tlap !ENV amsua_n18_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n18_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n18_channels + options: + channels: *amsua_n18_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n18_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n18_channels + +passedBenchmark: 137310 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/obs/testing/amsua_n19.yaml b/parm/atm/obs/testing/amsua_n19.yaml index f91b045c1..b0f3bb45e 100644 --- a/parm/atm/obs/testing/amsua_n19.yaml +++ b/parm/atm/obs/testing/amsua_n19.yaml @@ -1,12 +1,3 @@ -obs operator: - name: CRTM - Absorbers: [H2O,O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 - obs options: - Sensor_ID: &Sensor_ID amsua_n19 - EndianType: little_endian - CoefficientPath: crtm/ obs space: name: amsua_n19 obsdatain: @@ -18,9 +9,21 @@ obs space: type: H5File obsfile: !ENV amsua_n19_diag_${CDATE}.nc4 simulated variables: [brightnessTemperature] - channels: &all_channels 1-15 + channels: &amsua_n19_channels 1-15 + geovals: filename: !ENV amsua_n19_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n19 + EndianType: little_endian + CoefficientPath: crtm/ + obs bias: input file: !ENV amsua_n19_satbias_${GDATE}.nc4 variational bc: @@ -40,49 +43,40 @@ obs bias: order: 2 - name: scan_angle -obs post filters: +obs pre filters: # Step 0-A: Create Diagnostic Flags -# Diagnostic flag for CLW retrieval - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: CLWRetrievalReject - initial value: false - force reinitialization: true - -# Diagnostic flag for hydrometeor check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: HydrometeorCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for gross check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels - flags: - - name: GrossCheckReject - initial value: false - force reinitialization: true - -# Diagnostic flag for inter-channel consistency check -- filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *all_channels + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels flags: - - name: InterChannelCheckReject - initial value: false - force reinitialization: true + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false -# Step 0-B: Calculate derived variables +obs post filters: +# Step 0-B: Calculate Derived Variables # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -153,13 +147,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/CLWMatchIndexMW - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels clwobs_function: name: DerivedMetaData/CLWRetFromObs clwbkg_function: @@ -172,13 +166,13 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorModelRamp - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels xvar: name: DerivedMetaData/CLWRetSymmetric x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, @@ -198,29 +192,29 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/Innovation - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/Arithmetic - channels: *all_channels + channels: *amsua_n19_channels options: variables: - name: ObsValue/brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels - name: HofX/brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels coefs: [1, -1] -# Step 1: Assign initial all-sky observation error +# Step 1: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: assign error error function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels # Step 2: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -232,7 +226,7 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject # Step 3: CLW Retrieval Check (background_based) @@ -245,31 +239,35 @@ obs post filters: maxvalue: 999.0 actions: - name: set - flag: CLWRetrievalReject + flag: CLWRetrievalCheck - name: reject -# Step 4: Window channel sanity check +# Step 4: Window Channel Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature channels: 1-6, 15 test variables: - name: DerivedMetaData/Innovation - channels: 1, 2 4-6, 15 + channels: 1, 2, 4-6, 15 maxvalue: 200.0 minvalue: -200.0 flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels test variables: - name: ObsFunction/HydrometeorCheckAMSUA - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels obserr_clearsky: [ 2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -277,96 +275,96 @@ obs post filters: name: DerivedMetaData/CLWRetFromObs obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels maxvalue: 0.0 actions: - name: set - flag: HydrometeorCheckReject + flag: HydrometeorCheck ignore: rejected observations - name: reject -# Step 6: Observation error inflation based on topography check +# Step 6: Observation Error Inflation based on Topography Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorTopoRad - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n19_channels -# Step 7: Obs error inflation based on TOA transmittancec check +# Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n19_channels -# Step 8: Observation error inflation based on surface jacobian check +# Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSurfJacobian - channels: *all_channels + channels: *amsua_n19_channels -# Step 9: Situation dependent check +# Step 9: Situation Dependent Check - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorFactorSituDependMW - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels clwbkg_function: name: DerivedMetaData/CLWRetFromBkg clwobs_function: @@ -375,10 +373,10 @@ obs post filters: name: DerivedMetaData/SIRetFromObs clwmatchidx_function: name: DerivedMetaData/CLWMatchIndex - channels: *all_channels + channels: *amsua_n19_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels obserr_clearsky: [2.500, 2.200, 2.000, 0.550, 0.300, 0.230, 0.230, 0.250, 0.250, 0.350, 0.400, 0.550, 0.800, 4.000, 3.500] @@ -386,14 +384,14 @@ obs post filters: - filter: Perform Action filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels action: name: inflate error inflation variable: name: DerivedMetaData/ObsErrorFactorSituDepend - channels: *all_channels + channels: *amsua_n19_channels -# Step 10: Gross check +# Step 10: Gross Check # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: @@ -407,27 +405,27 @@ obs post filters: - filter: Variable Assignment assignments: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n19_channels type: float function: name: ObsFunction/ObsErrorBoundMW - channels: *all_channels + channels: *amsua_n19_channels options: sensor: *Sensor_ID - channels: *all_channels + channels: *amsua_n19_channels obserr_bound_latitude: name: DerivedMetaData/ObsErrorFactorLat obserr_bound_transmittop: name: DerivedMetaData/ObsErrorFactorTransmitTop - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels obserr_bound_topo: name: DerivedMetaData/ObsErrorFactorTopo - channels: *all_channels + channels: *amsua_n19_channels obserr_function: name: DerivedMetaData/InitialObsError - channels: *all_channels + channels: *amsua_n19_channels threhold: 3 obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -436,26 +434,26 @@ obs post filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels function absolute threshold: - name: DerivedMetaData/ObsErrorBound - channels: *all_channels + channels: *amsua_n19_channels actions: - name: set - flag: GrossCheckReject + flag: GrossCheck ignore: rejected observations - name: reject -# Step 11: Inter-channel check +# Step 11: Inter-Channel Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *all_channels + channels: *amsua_n19_channels test variables: - name: ObsFunction/InterChannelConsistencyCheck - channels: *all_channels + channels: *amsua_n19_channels options: - channels: *all_channels + channels: *amsua_n19_channels use passive_bc: true sensor: *Sensor_ID use_flag: [ 1, 1, 1, 1, 1, @@ -464,7 +462,29 @@ obs post filters: maxvalue: 1.0e-12 actions: - name: set - flag: InterChannelCheckReject + flag: InterChannelConsistency ignore: rejected observations - name: reject -passedBenchmark: 107747 + +# Step 12: Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + use_flag: [ 1, 1, 1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, 1 ] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +passedBenchmark: 94073 +#passedBenchmark: 106514 # no use flag check diff --git a/parm/atm/obs/testing/amsua_n19_noqc.yaml b/parm/atm/obs/testing/amsua_n19_noqc.yaml new file mode 100644 index 000000000..98c7efed5 --- /dev/null +++ b/parm/atm/obs/testing/amsua_n19_noqc.yaml @@ -0,0 +1,140 @@ +obs space: + name: amsua_n19 + obsdatain: + engine: + type: H5File + obsfile: !ENV amsua_n19_obs_${CDATE}.nc4 + obsdataout: + engine: + type: H5File + obsfile: !ENV amsua_n19_diag_${CDATE}.nc4 + simulated variables: [brightnessTemperature] + channels: &amsua_n19_channels 1-15 + +geovals: + filename: !ENV amsua_n19_geoval_${CDATE}.nc4 + +obs operator: + name: CRTM + Absorbers: [H2O,O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &Sensor_ID amsua_n19 + EndianType: little_endian + CoefficientPath: crtm/ + +obs bias: + input file: !ENV amsua_n19_satbias_${GDATE}.nc4 + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &amsua_n19_tlap !ENV amsua_n19_tlapmean_${GDATE}.txt + - name: lapse_rate + tlapse: *amsua_n19_tlap + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + +obs post filters: +# Step 0-B: Calculate derived variables +# Calculate CLW retrieved from observation +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromObs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + +# Calculate CLW retrieved from background +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetFromBkg + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + +# Calculate symmetric retrieved CLW +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + value: 1000.0 + +- filter: Variable Assignment + where: + - variable: + name: DerivedMetaData/CLWRetFromObs + minvalue: 0. + maxvalue: 999. + - variable: + name: DerivedMetaData/CLWRetFromBkg + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: DerivedMetaData/CLWRetSymmetric + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/CLWRetFromObs + - name: DerivedMetaData/CLWRetFromBkg + total coefficient: 0.5 + +# Calculate symmetric observation error +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + type: float + function: + name: ObsFunction/ObsErrorModelRamp + channels: *amsua_n19_channels + options: + channels: *amsua_n19_channels + xvar: + name: DerivedMetaData/CLWRetSymmetric + x0: [ 0.050, 0.030, 0.030, 0.020, 0.000, + 0.100, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.030] + x1: [ 0.600, 0.450, 0.400, 0.450, 1.000, + 1.500, 0.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.200] + err0: [ 2.500, 2.200, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 3.500] + err1: [20.000, 18.000, 12.000, 3.000, 0.500, + 0.300, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 4.000, 18.000] + +# Step 1: Assign initial all-sky observation error +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_channels + action: + name: assign error + error function: + name: DerivedMetaData/InitialObsError + channels: *amsua_n19_channels + +passedBenchmark: 136742 +#vector ref: GsiHofXBc +#tolerance: 1.e-7 diff --git a/parm/atm/variational/3dvar_drpcg.yaml.j2 b/parm/atm/variational/3dvar_drpcg.yaml.j2 index c0f684413..156877371 100644 --- a/parm/atm/variational/3dvar_drpcg.yaml.j2 +++ b/parm/atm/variational/3dvar_drpcg.yaml.j2 @@ -44,7 +44,7 @@ variational: minimizer: algorithm: DRPCG iterations: - - ninner: 2 + - ninner: 1 gradient norm reduction: 1e-10 test: on geometry: diff --git a/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc new file mode 100644 index 000000000..28b062add Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc new file mode 100644 index 000000000..fdc91fc80 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_metop-c_v2.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-c_v2.ACCoeff.nc new file mode 100644 index 000000000..c7a31156e Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_metop-c_v2.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc new file mode 100644 index 000000000..bf3a9c0f6 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc new file mode 100644 index 000000000..efeef51c7 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc new file mode 100644 index 000000000..a50cc26e0 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc new file mode 100644 index 000000000..72310903c Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc new file mode 100644 index 000000000..bc36197de Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml index 0f1d69fb7..532e8510c 100644 --- a/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml +++ b/parm/ioda/bufr2ioda/bufr2ioda_atms.yaml @@ -27,11 +27,8 @@ observations: satelliteInstrument: query: "*/SIID" - # The type should be integer. - # However, UFO/CRTM requred this variable to be float fieldOfViewNumber: query: "*/FOVN" -# type: float heightOfStation: query: "*/HMSL" diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml new file mode 100644 index 000000000..345d4bd65 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_1bamua_ta.yaml @@ -0,0 +1,190 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.1bamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + satelliteIdentifier: + query: "*/SAID" + + satelliteInstrument: + query: "*/SIID" + + fieldOfViewNumber: + query: "*/FOVN" + + landOrSeaQualifier: + query: "*/LSQL" + + heightOfLandSurface: + query: "*/HOLS" + + heightOfStation: + query: "*/HMSL" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + + sensorChannelNumber: + query: "*/BRITCSTC/CHNM" + + # ObsValue + # Note: BUFR Dump contains Antenna Temperature for all normal-feed AMSUAs except NOAA-15/16 + # NOAA-15/16 contain Brightness Temperature + antennaTemperature: + query: "*/BRITCSTC/TMBR" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c + _208: n17 + _209: n18 + _223: n19 + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.amsua.$(splitvar)_ta.tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/BRITCSTC" + + globals: + - name: "platformCommonName" + type: string + value: "AMSU-A" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-023 PROC AMSU-A 1B Tb" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteInstrument" + source: variables/satelliteInstrument + longName: "Satellite Instrument" + + - name: "MetaData/landOrSeaQualifier" + source: variables/landOrSeaQualifier + longName: "Land/Sea Qualifier" + + - name: "MetaData/heightOfLandSurface" + source: variables/heightOfLandSurface + longName: "Height of Land Surface" + units: "m" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorScanAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + - name: "MetaData/sensorScanPosition" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + # ObsValue + # For now, store the antennaTemperature as brightnessTemperature + - name: "ObsValue/brightnessTemperature" + source: variables/antennaTemperature + longName: "Antenna Temperature" + units: "K" + range: [90, 380] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json b/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json new file mode 100644 index 000000000..cdde43797 --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_amsua.json @@ -0,0 +1,21 @@ +{ + "data_format" : "bufr_d", + "data_type" : ["esamua", "1bamua_td"], + "cycle_type" : "{{ RUN }}", + "cycle_datetime" : "{{ current_cycle | to_YMDH }}", + "dump_directory" : "{{ DMPDIR }}", + "ioda_directory" : "{{ COM_OBS }}", + "subsets" : [ "NC005030", "NC005031", "NC005032", "NC005034", "NC005039" ], + "data_description" : "NC005030 NESDIS SATWIND, GOES IR(LW); NC005031 NESDIS SATWIND, GOES WV-IMG/DL; NC005032 NESDIS SATWIND, GOES VIS; NC005034 NESDIS SATWIND, GOES WV-IMG/CT; NC005039 NESDIS SATWIND, GOES IR(SW)", + "data_provider" : "U.S. NOAA/NDESDIS", + "sensor_info" : { "sensor_name": "ABI", "sensor_full_name": "Advanced Baseline Imager", "sensor_id": 617 }, + "satellite_info" : [ + { "satellite_name": "GOES-16", "satellite_full_name": "Geostationary Operational Satellite - 16", "satellite_id": 270 }, + { "satellite_name": "GOES-17", "satellite_full_name": "Geostationary Operational Satellite - 17", "satellite_id": 271 }, + { "satellite_name": "GOES-18", "satellite_full_name": "Geostationary Operational Satellite - 18", "satellite_id": 272 } + ], + "yaml_file" : ["{{ DATA }}/ncep_esamua_{{ current_cycle | to_YMDH }}.yaml", + "{{ DATA }}/ncep_1bamua_ta_{{ current_cycle | to_YMDH }}.yaml"], + "ac_dir" : "{{ DATA }}" + +} diff --git a/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml new file mode 100644 index 000000000..d577e94ce --- /dev/null +++ b/parm/ioda/bufr2ioda/bufr2ioda_ncep_esamua.yaml @@ -0,0 +1,187 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "{{ DMPDIR }}/{{ RUN }}.{{ PDY }}/{{ cyc }}/atmos/{{ RUN }}.t{{ cyc }}z.esamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteIdentifier: + query: "*/SAID" + + satelliteAntennaCorrectionsVersionNumber: + query: "*/SACV" + + orbitNumber: + query: "*/ORBN" + + scanLineNumber: + query: "*/SLNM" + + fieldOfViewNumber: + query: "*/FOVN" + + heightOfStation: + query: "*/SELV" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorScanAngle: + sensorScanAngle: + fieldOfViewNumber: "*/FOVN" + scanStart: -48.333 + scanStep: 3.333 + sensor: amsua + + sensorChannelNumber: + query: "*/ATNCHV/INCN" + + # ObsValue + brightnessTemperature: + query: "*/ATNCHV/TMBRST" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c +# _206: n15 + _209: n18 + _223: n19 + + ioda: + backend: netcdf + obsdataout: "{{ COM_OBS }}/{{ RUN }}.t{{ cyc }}z.esamua.$(splitvar).tm00.nc" + + dimensions: + - name: Channel + source: variables/sensorChannelNumber + path: "*/ATNCHV" + + globals: + - name: "platformCommonName" + type: string + value: "AMSUA" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb DATA)" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteAntennaCorrectionsVersionNumber" + source: variables/satelliteAntennaCorrectionsVersionNumber + longName: "Satellite Antenna Corrections Version Number" + + - name: "MetaData/sensorScanPosition" + source: variables/fieldOfViewNumber + longName: "Field Of View Number" + + - name: "MetaData/orbitNumber" + source: variables/orbitNumber + longName: "Orbit Number" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number1" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorViewAngle" + source: variables/sensorScanAngle + longName: "Sensor View Angle" + units: "degree" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemperature + longName: "Brightness Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py new file mode 100644 index 000000000..096223643 --- /dev/null +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -0,0 +1,42 @@ +import netCDF4 as nc +import os +import numpy as np + +R1000 = 1000.0 +R1000000 = 1000000.0 +INVALID = R1000 +# Cosmic background temperature. Taken from Mather,J.C. et. al., 1999, "Calibrator Design for the COBE +# Far-Infrared Absolute Spectrophotometer (FIRAS)"Astrophysical Journal, vol 512, pp 511-520 +TSPACE = 2.7253 + + +class ACCoeff: + def __init__(self, ac_dir, sat_id='n19'): + file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '.ACCoeff.nc') + nc_file = nc.Dataset(file_name) + self.n_fovs = len(nc_file.dimensions['n_FOVs']) + self.n_channels = len(nc_file.dimensions['n_Channels']) + self.a_earth = nc_file.variables['A_earth'][:] + self.a_platform = nc_file.variables['A_platform'][:] + self.a_space = nc_file.variables['A_space'][:] + self.a_ep = self.a_earth + self.a_platform + self.a_sp = self.a_space * TSPACE + + +def remove_ant_corr(i, ac, ifov, t): + # AC: Structure containing the antenna correction coefficients for the sensor of interest. + # iFOV: The FOV index for a scanline of the sensor of interest. + # T: On input, this argument contains the brightness + + # print(f't before corr: {t[:100]}') + t = ac.a_ep[i, ifov] * t + ac.a_sp[i, ifov] + # print(f't after corr: {t[:100]}') + t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID] + return t + + +def apply_ant_corr(i, ac, ifov, t): + # t: on input, this argument contains the antenna temperatures for the sensor channels. + t = (t - ac.a_sp[i, ifov]) / ac.a_ep[i, ifov] + t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID] + return t diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py new file mode 100755 index 000000000..7e14f740d --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +from combine_base import Bufr2IodaBase, CPP +from wxflow import Logger +from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr, R1000, R1000000 +from utils import timing_decorator, nc_merge + +logger = Logger(os.path.basename(__file__), level='INFO') + +BACKEND = CPP + +AMSUA_TYPE_CHANGE_DATETIME = "2023120000" + +BAMUA = '1BAMUA' +ESAMUA = 'ESAMUA' + +YAML_NORMAL = True # current as normal + + +class Bufr2IodaAmusa(Bufr2IodaBase): + def __init__(self, yaml_order, *args, **kwargs): + self.yaml_order = yaml_order + super().__init__(*args, **kwargs) + + def get_yaml_file(self): + if self.yaml_order: + return self.config['yaml_file'][0] + else: + return self.config['yaml_file'][1] + + +class Bufr2IodaAmusaChange(Bufr2IodaAmusa): + def get_yaml_file(self): + if self.yaml_order: + return self.config['yaml_file'][1] + else: + return self.config['yaml_file'][0] + + def get_ac_dir(self): + return self.config['ac_dir'] + + @timing_decorator + def re_map_variable(self, container): + # TODO replace this follow that in GSI + # read_bufrtovs.f90 + # antcorr_application.f90 + # search the keyword “ta2tb” for details + + for sat_id in self.sat_ids: + logger.info(f'Converting for {sat_id}, ...') + ta = self.get_container_variable(container, 'ObsValue', 'brightnessTemperature', sat_id) + if ta.shape[0]: + if self.yaml_order: + ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) + else: + ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) + logger.debug(f'ta before correction1: {ta[:100, :]}') + tb = self.apply_corr(sat_id, ta, ifov) + logger.debug(f'tb after correction1: {tb[:100, :]}') + self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) + + def apply_corr(self, sat_id, ta, ifov): + ac = ACCoeff(self.get_ac_dir()) # TODO add later + llll = 1 # TODO how to set this + if llll == 1: + if sat_id not in ['n15', 'n16']: + # Convert antenna temperature to brightness temperature + ifov = ifov.astype(int) - 1 + for i in range(ta.shape[1]): + logger.debug(f'inside loop for allpy ta to tb: i = {i}') + x = ta[:, i] + # logger.info(f'ta before correction: {x[:100]}') + if self.yaml_order: + x = apply_ant_corr(i, ac, ifov, x) + else: + x = remove_ant_corr(i, ac, ifov, x) + # logger.info(f'ta after correction: {x[:100]}') + x[x >= R1000] = R1000000 + ta[:, i] = x + else: + pass # TODO after know how to set llll + return ta + + +@timing_decorator +def merge(amsua_files, splits): + ioda_files = [(f'amsua.{x}_ta.tm00.ncc', f'esamua.{x}.tm00.ncc', f'amsua_{x}.tm00.nc') for x in splits] + logger.info(f'Ioda files: {ioda_files}') + file1 = [f for f in amsua_files[0].values()] + file2 = [f for f in amsua_files[1].values()] + obs_path_prefix = os.path.commonprefix([file1[0], file2[0]]) + logger.info(f'common prefix: {obs_path_prefix}') + for ioda_file_0, ioda_file_1, ioda_file_t in ioda_files: + try: + nc_merge(obs_path_prefix + ioda_file_0, + obs_path_prefix + ioda_file_1, + obs_path_prefix + ioda_file_t + ) + except FileNotFoundError as e: + logger.info(f'File not found exception: {e}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input JSON configuration', required=True) + parser.add_argument('-v', '--verbose', help='print debug logging information', + action='store_true') + args = parser.parse_args() + log_level = 'DEBUG' if args.verbose else 'INFO' + logger = Logger(os.path.basename(__file__), level=log_level) + amsua_files = [] + splits = set() + json_file_name = args.config + try: + with open(json_file_name, "r") as json_file: + config = json.load(json_file) + except FileNotFoundError as e: + logger.info(f'Json file not existed exception: {json_file_name} with error msg: {e}') + exit + + cycle_datetime = config["cycle_datetime"] + if cycle_datetime >= AMSUA_TYPE_CHANGE_DATETIME: + yaml_order = YAML_NORMAL + else: + yaml_order = not YAML_NORMAL + logger.info(f'yaml order is {yaml_order}') + BAMUA = '1BAMUA' + ESAMUA = 'ESAMUA' + for sat_type in [BAMUA, ESAMUA]: + logger.info(f'Processing sat type: {sat_type}') + if sat_type == BAMUA: + convert = Bufr2IodaAmusa(yaml_order, config, backend=BACKEND) + else: + convert = Bufr2IodaAmusaChange(yaml_order, config, backend=BACKEND) + + convert.execute() + amsua_files.append(convert.split_files) + logger.info('Converting amsua type {} done!'.format(sat_type)) + splits.update(set(convert.sat_ids)) + + logger.info('--start to merge--') + merge(amsua_files, splits) + logger.info('--Finished merge--') diff --git a/ush/ioda/bufr2ioda/combine_base.py b/ush/ioda/bufr2ioda/combine_base.py new file mode 100644 index 000000000..52be538ed --- /dev/null +++ b/ush/ioda/bufr2ioda/combine_base.py @@ -0,0 +1,120 @@ +import json +import os +import xarray as xr +import re +import yaml + +from pyiodaconv import bufr + +from wxflow import Logger + +from utils import timing_decorator + +CPP = 1 +PYTHON = 2 +GROUPS = ['MetaData', 'ObsValue'] + +logger = Logger(os.path.basename(__file__), level='DEBUG') + + +class Bufr2IodaBase: + + def __init__(self, config, backend=PYTHON): + self.config = config + yaml_config = self.get_yaml_config() + self.yaml_config = yaml_config + self.ioda_files = yaml_config['observations'][0]['ioda'].get('obsdataout') + logger.info(f'Ioda output files are: {self.ioda_files}') + self.splits = yaml_config['observations'][0]['obs space']['exports'].get('splits') + self.sat_ids = None + if self.splits: + self.sat_ids = [x for x in self.splits['satId']['category']['map'].values()] + logger.info(self.splits) + self.split_files = {} + self.yaml_path = None + self.backend = backend + + def get_container(self): + container = {} + if self.backend == PYTHON: + self.yaml_path = self.get_yaml_file() + container = bufr.Parser(self.yaml_path).parse() + elif self.backend == CPP: + container = {} + for sat_id in self.sat_ids: + logger.info(f'Processing sat_id: {sat_id}') + try: + container[sat_id] = {} + file_name = self.split_files[sat_id] + container[sat_id]['dset'] = xr.open_dataset(file_name) + for group in GROUPS: + container[sat_id][group] = xr.open_dataset(file_name, group=group, decode_times=False) + except FileNotFoundError as e: + logger.info(f'File not existed exception for sat id: {sat_id} with error msg: {e}') + container.pop(sat_id) + self.sat_ids = container.keys() + logger.info(f'sat_ids included in the container: {container.keys()}') + return container + + def get_container_variable(self, container, group, variable, sat_id): + ret = None + if self.backend == PYTHON: + ret = container.get(group + '/' + variable, sat_id) + elif self.backend == CPP: + ret = container[sat_id][group][variable] + return ret + + def replace_container_variable(self, container, group, variable, var, sat_id): + if self.backend == PYTHON: + container.replace(group + '/' + variable, sat_id) + elif self.backend == CPP: + container[sat_id][group][variable] = var + + def get_yaml_file(self): + return self.config['yaml_file'] + + def get_yaml_config(self): + yaml_file = self.get_yaml_file() + with open(yaml_file, 'r') as file: + yaml_config = yaml.load(file, Loader=yaml.FullLoader) + return yaml_config + + def make_split_files(self, sat_ids): + # sat_id a list of ids + pattern = r'{(.*?)}' + for sat_id in sat_ids: + self.split_files[sat_id] = re.sub(pattern, sat_id, self.ioda_files) + + def re_map_variable(self, container): + # Make any changes and return for your specific case in the sub-class + pass + + def ioda_encode(self, container): + if self.backend == PYTHON: + ioda_description = bufr.IodaDescription(self.yaml_path) + bufr.IodaEncoder(ioda_description).encode(container) + elif self.backend == CPP: + for sat_id in container: + logger.info(f'Encode for {sat_id}') + container[sat_id]['dset'].close() + file_name = self.split_files[sat_id] + 'c' + container[sat_id]['dset'].to_netcdf(file_name) + for group in GROUPS: + container[sat_id][group].to_netcdf(file_name, mode='a', group=group) + + @timing_decorator + def execute(self): + if self.splits: + self.make_split_files(self.sat_ids) + container = self.get_container() + self.re_map_variable(container) + self.ioda_encode(container) + + def get_info(self): + if not self.split_files: + self.make_split_files(self.splits) + sat_info = {'ioda_files': self.ioda_files, + 'split': self.splits, + 'split_files': self.split_files + } + return sat_info diff --git a/ush/ioda/bufr2ioda/run_bufr2ioda.py b/ush/ioda/bufr2ioda/run_bufr2ioda.py index 153cf2de5..3029a0ed2 100755 --- a/ush/ioda/bufr2ioda/run_bufr2ioda.py +++ b/ush/ioda/bufr2ioda/run_bufr2ioda.py @@ -25,7 +25,10 @@ def mp_bufr_converter(exename, configfile): cmd.add_default_arg('-c') cmd.add_default_arg(configfile) logger.info(f"Executing {cmd}") - cmd() + try: + cmd() + except Exception as e: + logger.info(f' Exception: {e}') @logit(logger) @@ -45,6 +48,7 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): config = { 'RUN': RUN, 'current_cycle': current_cycle, + 'DATA': DATA, 'DMPDIR': DMPDIR, 'COM_OBS': COM_OBS, 'PDY': current_cycle.strftime('%Y%m%d'), @@ -52,13 +56,23 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): } # copy necessary fix files to runtime directory - shutil.copy(os.path.join(config_template_dir, "atms_beamwidth.txt"), - os.path.join(DATA, "atms_beamwidth.txt")) + fix_files = ["atms_beamwidth.txt", + "amsua_n19.ACCoeff.nc" # TODO need to find a place for this .nc file. + ] + for fix_file in fix_files: + shutil.copy(os.path.join(config_template_dir, fix_file), os.path.join(DATA, fix_file)) + + # get all observation types to be processed by python + BUFR_total_py_files = set(glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py'))) + combine_type = 'bufr2ioda_combine_' + BUFR_combine_files = set(glob.glob(os.path.join(USH_IODA, combine_type + '*.py'))) # Specify observation types to be processed by a script - BUFR_py_files = glob.glob(os.path.join(USH_IODA, 'bufr2ioda_*.py')) + BUFR_py_files = BUFR_total_py_files.difference(BUFR_combine_files) + BUFR_py_files = [os.path.basename(f) for f in BUFR_py_files] BUFR_py = [f.replace('bufr2ioda_', '').replace('.py', '') for f in BUFR_py_files] + logger.info(f'All obs type processed by python: {BUFR_py}') config_files = [] exename = [] @@ -73,8 +87,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): bufr2iodapy = USH_IODA + '/bufr2ioda_' + obtype + ".py" # append the values to the lists - config_files.append(json_output_file) - exename.append(bufr2iodapy) + # config_files.append(json_output_file) + # exename.append(bufr2iodapy) # Check if the converter was successful # if os.path.exists(json_output_file): @@ -84,6 +98,8 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): BUFR_yaml_files = glob.glob(os.path.join(config_template_dir, '*.yaml')) BUFR_yaml_files = [os.path.basename(f) for f in BUFR_yaml_files] BUFR_yaml = [f.replace('bufr2ioda_', '').replace('.yaml', '') for f in BUFR_yaml_files] + logger.info(f'All obs type processed by yaml: {BUFR_yaml}') + BUFR_yaml = ['ncep_1bamua_ta', 'ncep_esamua'] for obtype in BUFR_yaml: logger.info(f"Convert {obtype}...") @@ -103,6 +119,31 @@ def bufr2ioda(current_cycle, RUN, DMPDIR, config_template_dir, COM_OBS): # if os.path.exists(yaml_output_file): # rm_p(yaml_output_file) + # run everything in parallel + with mp.Pool(num_cores) as pool: + pool.starmap(mp_bufr_converter, zip(exename, config_files)) + + config_files = [] + exename = [] + # Specify observation types to be processed by a script with combined methods + BUFR_combine_files = [os.path.basename(f) for f in BUFR_combine_files] + BUFR_combine = [f.replace(combine_type, '').replace('.py', '') for f in BUFR_combine_files] + logger.info(f'All obs type processed by combine: {BUFR_combine}') + + for obtype in BUFR_combine: + logger.info(f"Convert {obtype}...") + json_output_file = os.path.join(DATA, f"{obtype}_{datetime_to_YMDH(current_cycle)}.json") + filename = 'bufr2ioda_' + obtype + '.json' + template = os.path.join(config_template_dir, filename) + gen_bufr_json(config, template, json_output_file) + + # Use the converter script for the ob type + bufr2iodapy = os.path.join(USH_IODA, combine_type + obtype + ".py") + + # append the values to the lists + config_files.append(json_output_file) + exename.append(bufr2iodapy) + # run everything in parallel with mp.Pool(num_cores) as pool: pool.starmap(mp_bufr_converter, zip(exename, config_files)) diff --git a/ush/ioda/bufr2ioda/utils.py b/ush/ioda/bufr2ioda/utils.py new file mode 100644 index 000000000..76c3b7ba0 --- /dev/null +++ b/ush/ioda/bufr2ioda/utils.py @@ -0,0 +1,90 @@ +import netCDF4 as nc +import numpy as np +import os +import time +from wxflow import Logger +logger = Logger(os.path.basename(__file__), level='INFO') + + +def nc_merge(file_name1, file_name2, target_name): + ncf1 = nc.Dataset(file_name1) + ncf2 = nc.Dataset(file_name2) + ncf = nc.Dataset(target_name, "w", format="NETCDF4") + + # Add attr for dataset + ncf1_attr = ncf1.__dict__ + ncf2_attr = ncf2.__dict__ + ncf_attr = {} + for k, v1 in ncf1_attr.items(): + ncf_attr[k] = v1 + v2 = ncf2_attr.get(k) + if v2 and v1 != v2: + ncf_attr[k] = f"1: {v1}, 2: {v2}" + ncf.setncatts(ncf_attr) + + # Create groups + for grp in ncf1.groups: + logger.info(grp) + ncf.createGroup(grp) + + # Create dimensions + ncf1_dims = ncf1.dimensions + ncf2_dims = ncf2.dimensions + for dim_key in ncf1_dims: + dim1 = ncf1_dims[dim_key] + dim2 = ncf2_dims.get(dim_key) + if dim2: + if dim1.name == 'Location': + dim_size = dim1.size + dim2.size + else: + dim_size = dim1.size + ncf.createDimension(dim1.name, dim_size) + else: + logger.info('error: dimension mismatch') # TODO throw an exception + + # Create Dimension variables + vars2 = ncf2.variables + for var1 in ncf1.variables.values(): + if var1.name == 'Channel': + var = var1 + else: + var = np.concatenate((var1, vars2[var1.name])) + ncf_var = ncf.createVariable(var1.name, var1.datatype, var1.dimensions) + ncf_var[:] = var[:] + attrs = var1.ncattrs() + for attr in attrs: + ncf_var.setncattr(attr, var1.getncattr(attr)) + + # Create Group variables + grp1 = ncf1.groups + grp2 = ncf2.groups + for k in grp1.keys(): + vars1 = grp1[k].variables + vars2 = grp2[k].variables + for key in vars1: + var = vars1[key] + if vars2.get(key): + var2 = vars2[key] + else: + logger.info(f"-----------Warning: {key} is not existed in ncf2, skip -----------------") + continue + var_array = np.concatenate((var, var2)) + var_path = f"{grp1[k].path}/{var.name}" + var_fill_value = var.getncattr('_FillValue') + ncf_var = ncf.createVariable(var_path, var.datatype, var.dimensions, fill_value=var_fill_value) + ncf_var[:] = var_array[:] + attrs = var.ncattrs() + for attr in attrs: + if attr != '_FillValue': + ncf_var.setncattr(attr, var.getncattr(attr)) + + +def timing_decorator(func): + def wrapper(*args, **kwargs): + start_time = time.time() + result = func(*args, **kwargs) + end_time = time.time() + execution_time = end_time - start_time + logger.info(f"Function '{func.__name__}' took {execution_time:.6f} seconds to execute.") + return result + return wrapper