Feature/rtofs in situ

utils/obsproc/CMakeLists.txt

@@ -1 +1,25 @@
+
+add_subdirectory(rtofs)
+
+# add_library(obsproc STATIC
+	# Ghrsst2Ioda.h
+	# IcecAmsr2Ioda.h
+	# NetCDFToIodaConverter.h
+	# Rads2Ioda.h
+	# RTOFSInSitu.h
+	# Smap2Ioda.h
+	# Smos2Ioda.h
+	# superob.h
+	# util.h
+	# Viirsaod2Ioda.h
+# )
+# target_compile_features(
+	# onsproc
+	# PUBLIC cxx_std_17
+# )
+# target_link_libraries(
+	# obsproc
+	# PUBLIC oops ioda NetCDF::NetCDF_CXX
+# )
+
 add_subdirectory(applications)

utils/obsproc/RTOFSInSitu.h

@@ -0,0 +1,107 @@
+#pragma once
+
+#include <iostream>
+#include <netcdf>    // NOLINT (using C API)
+#include <regex>
+#include <string>
+#include <vector>
+
+#include "eckit/config/LocalConfiguration.h"
+
+#include <Eigen/Dense>    // NOLINT
+
+#include "ioda/Group.h"
+#include "ioda/ObsGroup.h"
+
+#include "NetCDFToIodaConverter.h"
+
+#include "rtofs/RTOFSDataFile.h"
+#include "rtofs/RTOFSOb.h"
+
+
+namespace gdasapp
+{
+
+
+class RTOFSInSitu:
+    public NetCDFToIodaConverter
+{
+ public:
+    explicit RTOFSInSitu(
+        const eckit::Configuration & fullConfig,
+        const eckit::mpi::Comm & comm):
+        NetCDFToIodaConverter(fullConfig, comm)
+    {
+        variable_ = "temperature";
+    }
+
+    // Read binary file and populate iodaVars
+    gdasapp::obsproc::iodavars::IodaVars
+    providerToIodaVars(const std::string fileName) final
+    {
+        oops::Log::info()
+            << "Processing files provided by RTOFS"
+            << std::endl;
+
+        // Set the int metadata names
+        std::vector<std::string> intMetadataNames = {"temperature"};
+
+        // Set the float metadata name
+        std::vector<std::string> floatMetadataNames = {};
+
+        // read the file
+        oops::Log::info()
+            << "Processing file "
+            << fileName
+            << std::endl;
+
+        rtofs::RTOFSDataFile RTOFSFile(fileName);
+        int n = RTOFSFile.NumberOfObservations();
+        rtofs::RTOFSOb & ob = RTOFSFile.observations();
+
+        int NumberOfTemperatureValues = 0;
+        for (int i = 0;  i < n;  i ++)
+            NumberOfTemperatureValues += ob.lt[i];
+
+        gdasapp::obsproc::iodavars::IodaVars iodaVars(
+            NumberOfTemperatureValues,
+            floatMetadataNames,
+            intMetadataNames);
+        iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z";
+
+        // debugging output:
+        // std::string DataDirectory =
+        // "/home/edwardg/da/edwardg/global-workflow/sorc/gdas.cd/build/gdas-utils/test/obsproc/RTOFSdata";
+        // ob.print(DataDirectory);
+
+// oops::Log::debug() << "--- iodaVars.location_: " << iodaVars.location_  << std::endl;
+
+        int k = 0;
+        for (int i = 0;  i < n;  i ++)
+        for (int j = 0;  j < ob.lt[i];  j ++)
+        {
+            iodaVars.longitude_(k) = ob.lon[i];
+            iodaVars.latitude_(k) = ob.lat[i];
+            iodaVars.obsVal_(k) = ob.tmp[i][j];
+            iodaVars.obsError_(k) = ob.tmp_err[i][j];
+            iodaVars.datetime_(k) = ob.dtg[i];
+            // iodaVars.preQc_(k) = oneDimFlagsVal[i];
+            // iodaVars.intMetadata_.row(k) << -999;
+
+            k++;
+        }
+
+        // basic test for iodaVars.trim
+/*
+        Eigen::Array<bool, Eigen::Dynamic, 1> mask
+            = (iodaVars.obsVal_ > 0.0);
+        iodaVars.trim(mask);
+*/
+
+
+        return iodaVars;
+    };    // providerToIodaVars
+};  // class RTOFSInSitu
+
+
+}  // namespace gdasapp

utils/obsproc/applications/CMakeLists.txt

@@ -1,9 +1,16 @@
 list( APPEND gdasapp_provider2ioda_src_files
-  gdas_obsprovider2ioda.cc
-  gdas_obsprovider2ioda.h
-  )
+	gdas_obsprovider2ioda.cc
+	gdas_obsprovider2ioda.h
+)
+
 ecbuild_add_executable( TARGET gdas_obsprovider2ioda.x
                         SOURCES ${gdasapp_provider2ioda_src_files} )
 
 target_compile_features( gdas_obsprovider2ioda.x PUBLIC cxx_std_17)
+
 target_link_libraries( gdas_obsprovider2ioda.x PUBLIC oops ioda NetCDF::NetCDF_CXX)
+
+# message("CMAKE_SOURCE_DIR = ${CMAKE_SOURCE_DIR}")
+link_directories(${CMAKE_SOURCE_DIR}/rtofs)
+target_link_libraries( gdas_obsprovider2ioda.x PRIVATE rtofs)
+

utils/obsproc/applications/gdas_obsprovider2ioda.h

@@ -9,6 +9,7 @@
 #include "../Ghrsst2Ioda.h"
 #include "../IcecAmsr2Ioda.h"
 #include "../Rads2Ioda.h"
+#include "../RTOFSInSitu.h"
 #include "../Smap2Ioda.h"
 #include "../Smos2Ioda.h"
 #include "../Viirsaod2Ioda.h"
@@ -31,6 +32,9 @@ namespace gdasapp {
       } else if (provider == "GHRSST") {
         Ghrsst2Ioda conv2ioda(fullConfig, this->getComm());
         conv2ioda.writeToIoda();
+      } else if (provider == "RTOFS") {
+        RTOFSInSitu conv2ioda(fullConfig, this->getComm());
+        conv2ioda.writeToIoda();
       } else if (provider == "SMAP") {
         Smap2Ioda conv2ioda(fullConfig, this->getComm());
         conv2ioda.writeToIoda();

utils/obsproc/rtofs/CMakeLists.txt

@@ -0,0 +1,38 @@
+set(
+	rtofs_src_files
+	RTOFSDataFile.h
+	RTOFSDataFile.cc
+	RTOFSOb.h
+	RTOFSOb.cc
+	util.cc
+	util.h
+)
+
+list(
+	APPEND gdasapp_provider2ioda_src_files
+	${rtofs_src_files}
+)
+
+
+add_library(rtofs STATIC
+	${rtofs_src_files}
+)
+
+target_compile_features(
+	rtofs
+	PUBLIC cxx_std_17
+)
+
+# target_link_libraries(
+	# rtofs
+	# PUBLIC oops ioda NetCDF::NetCDF_CXX
+# )
+
+
+# ecbuild_add_executable(
+	# TARGET gdas_obsprovider2ioda.x
+	# SOURCES ${gdasapp_provider2ioda_src_files}
+# )
+
+# target_compile_features( gdas_obsprovider2ioda.x PUBLIC cxx_std_17)
+# target_link_libraries( gdas_obsprovider2ioda.x PUBLIC oops ioda NetCDF::NetCDF_CXX)

utils/obsproc/rtofs/README

@@ -0,0 +1,11 @@
+Author: E. Givelberg
+May 15, 2024
+
+The code in this rtofs directory reads a binary Fortran
+file and generates a data object that can be ingested into ioda.
+
+The fortran binary file consists of records, including additional
+information that needs to be skipped.
+There are typically 8 bytes to be skipped between 2 data arrays.
+The read data is converted from the big endian to the host
+format, which is little endian in linux.

utils/obsproc/rtofs/RTOFSDataFile.cc

@@ -0,0 +1,114 @@
+#include "RTOFSDataFile.h"
+
+#include <ctime>
+
+#include <iostream>
+using std::cerr;
+using std::endl;
+using std::cout;
+#include <iomanip>        // std::get_time
+
+#include <sstream>
+#include <string>
+
+#include "RTOFSOb.h"
+#include "util.h"
+
+
+namespace rtofs
+{
+
+typedef char char12[12];
+typedef char char7[7];
+
+
+std::time_t
+    SecondsSinceReferenceTime(char12 time)
+{
+    std::tm referenceTime = {};
+    referenceTime.tm_year = 70;  // 1970
+    referenceTime.tm_mon = 0;    // January (months are 0-based)
+    referenceTime.tm_mday = 1;   // 1st day of the month
+    referenceTime.tm_hour = 0;
+    referenceTime.tm_min = 0;
+    referenceTime.tm_sec = 0;
+    std::time_t referenceTimestamp = std::mktime(&referenceTime);
+
+    std::tm t = {};
+    std::istringstream ss(time);
+    ss >> std::get_time(&t, "%Y%m%d%H%M");
+
+    std::time_t timestamp = std::mktime(&t);
+    return std::difftime(timestamp, referenceTimestamp);
+}    // SecondsSinceReferenceTime
+
+
+
+RTOFSDataFile::
+RTOFSDataFile(std::string filename):
+    filename(filename)
+{
+    if (!file_exists(filename))
+    {
+        cerr << "File not found" << endl;
+        exit(1);
+    }
+
+
+    const char * fn = filename.c_str();
+    f = fopen(fn, "rb");
+    if (!f)
+    {
+        cerr << "Error opening file " << fn << endl;
+        exit(1);
+    }
+
+    read_file();
+}    // RTOFSDataFile::RTOFSDataFile
+
+
+void
+RTOFSDataFile::
+    read_file()
+{
+    fseek(f, 4, SEEK_CUR);
+
+    int n_read = read_int(f);
+    int mx_lvl = read_int(f);
+    int vrsn = read_int(f);
+
+    ob = new RTOFSOb(n_read, mx_lvl, vrsn);
+    nobs = n_read;
+
+    ob->read(f);
+
+    skip8bytes(f);
+
+    char12 * ob_dtg = new char12[n_read];
+    fread(ob_dtg, sizeof(char[12]), n_read, f);
+    for (int i = 0;  i < n_read;  i ++)
+        ob->dtg[i] = SecondsSinceReferenceTime(ob_dtg[i]);
+
+    skip8bytes(f);
+
+    char12 * ob_rct = new char12[n_read];
+    fread(ob_rct, sizeof(char[12]), n_read, f);
+
+    skip8bytes(f);
+
+    char7 * ob_sgn = new char7[n_read];
+    fread(ob_sgn, sizeof(char[7]), n_read, f);
+
+    if (vrsn == 2)
+    {
+        float ** glb_sal = new float * [n_read];
+        for (int i = 0;  i < n_read;  i ++)
+        {
+            int k = ob->lt[i];
+            glb_sal[i] = alloc_read_float_array(f, k);
+        }
+    }
+}    // read_file
+
+
+}    // namespace rtofs

utils/obsproc/rtofs/RTOFSDataFile.h

@@ -0,0 +1,32 @@
+#ifndef BUNDLE_GDAS_UTILS_OBSPROC_RTOFS_RTOFSDATAFILE_H_
+#define BUNDLE_GDAS_UTILS_OBSPROC_RTOFS_RTOFSDATAFILE_H_
+
+#include <string>
+
+
+namespace rtofs
+{
+
+class RTOFSOb;
+
+class RTOFSDataFile
+{
+ public:
+    explicit RTOFSDataFile(std::string filename);
+    int NumberOfObservations() { return nobs; }
+    RTOFSOb & observations() { return * ob; }
+
+
+ private:
+    std::string filename;
+    int nobs;
+    FILE * f;
+    RTOFSOb * ob;
+
+    void read_file();
+};    // class RTOFSDataFile
+
+}    // namespace rtofs
+
+
+#endif    // BUNDLE_GDAS_UTILS_OBSPROC_RTOFS_RTOFSDATAFILE_H_