From 99b172ae2cef2db98a9cbf9637a4327a334efe95 Mon Sep 17 00:00:00 2001 From: Mindo Choi Date: Fri, 5 Apr 2024 18:29:13 +0000 Subject: [PATCH 1/4] Resove the FillValue issue in calculation --- utils/obsproc/Ghrsst2Ioda.h | 40 ++++++++++++----------------- utils/test/testref/ghrsst2ioda.test | 6 ++--- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/utils/obsproc/Ghrsst2Ioda.h b/utils/obsproc/Ghrsst2Ioda.h index 7f0082da8..789427c3c 100644 --- a/utils/obsproc/Ghrsst2Ioda.h +++ b/utils/obsproc/Ghrsst2Ioda.h @@ -30,6 +30,8 @@ namespace gdasapp { oops::Log::info() << "Processing files provided by GHRSST" << std::endl; // Get the sst bounds from the configuration + std::string sstUnits; + fullConfig_.get("bounds.units", sstUnits); float sstMin; fullConfig_.get("bounds.min", sstMin); float sstMax; @@ -86,6 +88,8 @@ namespace gdasapp { ncFile.getVar("sst_dtime").getVar(sstdTime.data()); float dtimeScaleFactor; ncFile.getVar("sst_dtime").getAtt("scale_factor").getValues(&dtimeScaleFactor); + int32_t dtimeFillValue; + ncFile.getVar("sst_dtime").getAtt("_FillValue").getValues(&dtimeFillValue); // Read SST ObsValue std::vector sstObsVal(dimTime*dimLat*dimLon); @@ -143,9 +147,13 @@ namespace gdasapp { // TODO(Somebody): add sampled std. dev. of sst to the total obs error obserror[i][j] = static_cast(sstObsErr[index]) * errScaleFactor + errOffSet; - // epoch time in seconds - seconds[i][j] = static_cast(sstdTime[index]) * dtimeScaleFactor - + static_cast(refTime[0]); + // epoch time in seconds and avoid to use FillValue in calculation + if (sstdTime[index] == dtimeFillValue) { + seconds[i][j] = 0; + } else { + seconds[i][j] = static_cast(sstdTime[index]) * dtimeScaleFactor + + static_cast(refTime[0]); + } index++; } } @@ -157,36 +165,21 @@ namespace gdasapp { std::vector> lon2d_s; std::vector> lat2d_s; std::vector> obserror_s; + std::vector> seconds_s; if ( fullConfig_.has("binning") ) { sst_s = gdasapp::superobutils::subsample2D(sst, mask, fullConfig_); lon2d_s = gdasapp::superobutils::subsample2D(lon2d, mask, fullConfig_); lat2d_s = gdasapp::superobutils::subsample2D(lat2d, mask, fullConfig_); obserror_s = gdasapp::superobutils::subsample2D(obserror, mask, fullConfig_); + seconds_s = gdasapp::superobutils::subsample2D(seconds, mask, fullConfig_); } else { sst_s = sst; lon2d_s = lon2d; lat2d_s = lat2d; obserror_s = obserror; + seconds_s = seconds; } - // Non-Superobing part only applied to datetime - // TODO(Mindo): Refactor to make superob capable of processing all data - // TODO(ASGM) : Remove datetime mean when the time reading is fixed(L146) - // Compute the sum of valid obs values where mask == 1 - int64_t sum = 0; - int count = 0; - for (size_t i = 0; i < seconds.size(); ++i) { - for (size_t j = 0; j < seconds[0].size(); ++j) { - if (mask[i][j] == 1) { - sum += seconds[i][j]; - count++; - } - } - } - // Calculate the average and store datetime - // Replace the seconds_s to 0 when count is zero - int64_t seconds_s = count != 0 ? static_cast(sum / count) : 0; - // number of obs after subsampling int nobs = sst_s.size() * sst_s[0].size(); @@ -211,8 +204,7 @@ namespace gdasapp { iodaVars.obsVal_(loc) = sst_s[i][j]; iodaVars.obsError_(loc) = obserror_s[i][j]; iodaVars.preQc_(loc) = 0; - // Store averaged datetime (seconds) - iodaVars.datetime_(loc) = seconds_s; + iodaVars.datetime_(loc) = seconds_s[i][j]; // Store optional metadata, set ocean basins to -999 for now iodaVars.intMetadata_.row(loc) << -999; loc += 1; @@ -221,7 +213,7 @@ namespace gdasapp { // Basic QC Eigen::Array boundsCheck = - (iodaVars.obsVal_ > sstMin && iodaVars.obsVal_ < sstMax && iodaVars.datetime_ > 0.0); + (iodaVars.obsVal_ > sstMin && iodaVars.obsVal_ < sstMax && iodaVars.datetime_ > 0); iodaVars.trim(boundsCheck); return iodaVars; diff --git a/utils/test/testref/ghrsst2ioda.test b/utils/test/testref/ghrsst2ioda.test index 75a0764e4..1413f9d04 100644 --- a/utils/test/testref/ghrsst2ioda.test +++ b/utils/test/testref/ghrsst2ioda.test @@ -21,6 +21,6 @@ latitude: Max: 77.95 Sum: 623.423 datetime: - Min: 1269445063 - Max: 1269445063 - Sum: 10155560504 + Min: 1269445504 + Max: 1269445504 + Sum: 10155564032 From d1e291df83fa0f21bd6bf96d6f997388bf406c87 Mon Sep 17 00:00:00 2001 From: Mindo Choi Date: Fri, 5 Apr 2024 18:40:45 +0000 Subject: [PATCH 2/4] Remove old yaml requirements --- utils/obsproc/Ghrsst2Ioda.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/utils/obsproc/Ghrsst2Ioda.h b/utils/obsproc/Ghrsst2Ioda.h index 789427c3c..0200e2198 100644 --- a/utils/obsproc/Ghrsst2Ioda.h +++ b/utils/obsproc/Ghrsst2Ioda.h @@ -30,9 +30,7 @@ namespace gdasapp { oops::Log::info() << "Processing files provided by GHRSST" << std::endl; // Get the sst bounds from the configuration - std::string sstUnits; - fullConfig_.get("bounds.units", sstUnits); - float sstMin; + float sstMin; fullConfig_.get("bounds.min", sstMin); float sstMax; fullConfig_.get("bounds.max", sstMax); @@ -213,7 +211,7 @@ namespace gdasapp { // Basic QC Eigen::Array boundsCheck = - (iodaVars.obsVal_ > sstMin && iodaVars.obsVal_ < sstMax && iodaVars.datetime_ > 0); + (iodaVars.obsVal_ > sstMin && iodaVars.obsVal_ < sstMax && iodaVars.datetime_ > 0.0); iodaVars.trim(boundsCheck); return iodaVars; From 5a23b35ea9a2f0ed61a780c4cd2c08dbee876f44 Mon Sep 17 00:00:00 2001 From: Mindo Choi Date: Fri, 5 Apr 2024 18:49:39 +0000 Subject: [PATCH 3/4] Coding Norms --- utils/obsproc/Ghrsst2Ioda.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/obsproc/Ghrsst2Ioda.h b/utils/obsproc/Ghrsst2Ioda.h index 0200e2198..c83c877df 100644 --- a/utils/obsproc/Ghrsst2Ioda.h +++ b/utils/obsproc/Ghrsst2Ioda.h @@ -30,7 +30,7 @@ namespace gdasapp { oops::Log::info() << "Processing files provided by GHRSST" << std::endl; // Get the sst bounds from the configuration - float sstMin; + float sstMin; fullConfig_.get("bounds.min", sstMin); float sstMax; fullConfig_.get("bounds.max", sstMax); From c14ee373f4691abf574362e8947d2ff7b765835d Mon Sep 17 00:00:00 2001 From: Mindo Choi Date: Thu, 11 Apr 2024 19:30:59 +0000 Subject: [PATCH 4/4] type change --- utils/obsproc/Ghrsst2Ioda.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/obsproc/Ghrsst2Ioda.h b/utils/obsproc/Ghrsst2Ioda.h index c83c877df..fa277a8e8 100644 --- a/utils/obsproc/Ghrsst2Ioda.h +++ b/utils/obsproc/Ghrsst2Ioda.h @@ -120,7 +120,7 @@ namespace gdasapp { std::vector> sst(dimLat, std::vector(dimLon)); std::vector> obserror(dimLat, std::vector(dimLon)); std::vector> preqc(dimLat, std::vector(dimLon)); - std::vector> seconds(dimLat, std::vector(dimLon)); + std::vector> seconds(dimLat, std::vector(dimLon)); size_t index = 0; for (int i = 0; i < dimLat; i++) { for (int j = 0; j < dimLon; j++) { @@ -149,8 +149,8 @@ namespace gdasapp { if (sstdTime[index] == dtimeFillValue) { seconds[i][j] = 0; } else { - seconds[i][j] = static_cast(sstdTime[index]) * dtimeScaleFactor - + static_cast(refTime[0]); + seconds[i][j] = static_cast(sstdTime[index]) * dtimeScaleFactor + + static_cast(refTime[0]); } index++; } @@ -163,7 +163,7 @@ namespace gdasapp { std::vector> lon2d_s; std::vector> lat2d_s; std::vector> obserror_s; - std::vector> seconds_s; + std::vector> seconds_s; if ( fullConfig_.has("binning") ) { sst_s = gdasapp::superobutils::subsample2D(sst, mask, fullConfig_); lon2d_s = gdasapp::superobutils::subsample2D(lon2d, mask, fullConfig_);