Feature/query cxx types

src/bufr/BufrParser/BufrParser.cpp

@@ -66,9 +66,8 @@ namespace Ingester {
         {
             for (const auto& queryInfo : var->getQueryList())
             {
-                auto resultBase = resultSet.get(
-                        queryInfo.name, queryInfo.groupByField);
-                srcData[queryInfo.name] = DataObjectBase::fromResult(resultBase, queryInfo.query);
+                srcData[queryInfo.name] = resultSet.get(
+                    queryInfo.name, queryInfo.groupByField, queryInfo.type);
             }
         }
 

src/bufr/BufrParser/Exports/Export.cpp

@@ -32,9 +32,9 @@ namespace
         namespace Variable
         {
             const char* Datetime = "datetime";
-            const char* Mnemonic = "mnemonic";
             const char* Query = "query";
             const char* GroupByField = "group_by";  // Deprecated
+            const char* Type = "type";
         }  // namespace Variable
 
         namespace Split
@@ -125,13 +125,6 @@ namespace Ingester
                 auto dtconf = subConf.getSubConfiguration(ConfKeys::Variable::Datetime);
                 variable = std::make_shared<DatetimeVariable>(key, groupByField, dtconf);
             }
-            else if (subConf.has(ConfKeys::Variable::Mnemonic))
-            {
-                std::ostringstream errMsg;
-                errMsg << "Obsolete format::exports::variable of type " << key << std::endl;
-                errMsg << "Use \"query:\" instead.";
-                throw eckit::BadParameter(errMsg.str());
-            }
             else if (subConf.has(ConfKeys::Variable::Query))
             {
                 Transforms transforms = TransformBuilder::makeTransforms(subConf);
@@ -146,9 +139,16 @@ namespace Ingester
                     throw eckit::BadParameter(errMsg.str());
                 }
 
+                std::string type = "";
+                if (subConf.has(ConfKeys::Variable::Type))
+                {
+                    type = subConf.getString(ConfKeys::Variable::Type);
+                }
+
                 variable = std::make_shared<QueryVariable>(key,
                                                            query,
                                                            groupByField,
+                                                           type,
                                                            transforms);
             }
             else

src/bufr/BufrParser/Exports/Variables/QueryVariable.cpp

@@ -19,10 +19,12 @@ namespace Ingester
     QueryVariable::QueryVariable(const std::string& exportName,
                                  const std::string& query,
                                  const std::string& groupByField,
+                                 const std::string& type,
                                  const Transforms& transforms) :
         Variable(exportName),
         query_(query),
         groupByField_(groupByField),
+        type_(type),
         transforms_(transforms)
     {
         initQueryMap();
@@ -56,6 +58,7 @@ namespace Ingester
         info.name = getExportName();
         info.query = query_;
         info.groupByField = groupByField_;
+        info.type = type_;
         queries.push_back(info);
 
         return queries;

src/bufr/BufrParser/Exports/Variables/QueryVariable.h

@@ -28,6 +28,7 @@ namespace Ingester
         explicit QueryVariable(const std::string& exportName,
                                const std::string& query,
                                const std::string& groupByField,
+                               const std::string& type,
                                const Transforms& transforms);
 
         ~QueryVariable() final = default;
@@ -46,6 +47,9 @@ namespace Ingester
         /// \brief The for field of interest
         std::string groupByField_;
 
+        /// \brief Optional type override string
+        std::string type_;
+
         /// \brief Collection of transforms to apply to the data during export
         Transforms transforms_;
     };

src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.cpp

@@ -19,16 +19,7 @@ namespace Ingester
 
     void OffsetTransform::apply(std::shared_ptr<DataObjectBase>& dataObject)
     {
-      if (auto object = std::dynamic_pointer_cast<DataObject<float>>(dataObject))
-      {
-        auto data = object->getRawData();
-        for (auto& val : data)
-        {
-            if (val != bufr::MissingValue) val += offset_;
-        }
-
-        object->setRawData(data);
-      }
+        dataObject->offsetBy(offset_);
     }
 
 }  // namespace Ingester

src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.cpp

@@ -18,15 +18,6 @@ namespace Ingester
 
     void ScalingTransform::apply(std::shared_ptr<DataObjectBase>& dataObject)
     {
-      if (auto object = std::dynamic_pointer_cast<DataObject<float>>(dataObject))
-      {
-        auto data = object->getRawData();
-        for (auto& val : data)
-        {
-            if (val != bufr::MissingValue) val *= scaling_;
-        }
-
-        object->setRawData(data);
-      }
+        dataObject->multiplyBy(scaling_);
     }
 }  // namespace Ingester

src/bufr/BufrParser/Exports/Variables/Variable.h

@@ -21,6 +21,7 @@ namespace Ingester
         std::string name;
         std::string query;
         std::string groupByField;
+        std::string type;
     };
 
     typedef std::string QueryName;

src/bufr/BufrParser/Query/Constants.h

@@ -9,6 +9,6 @@
 namespace Ingester {
 namespace bufr {
     /// \brief The missing data value for all BUFR data.
-    const float MissingValue = 10e10;
+    const double MissingValue = 10.0e10;
 }  // Ingester
 }  // bufr

src/bufr/BufrParser/Query/DataProvider.cpp

@@ -12,6 +12,8 @@
 #include <iostream>
 #include <unordered_map>
 
+#include "eckit/exception/Exceptions.h"
+
 namespace
 {
     const char* Subset = "SUB";
@@ -32,6 +34,7 @@ namespace bufr {
 
     void DataProvider::updateData(int bufrLoc)
     {
+        bufrLoc_ = bufrLoc;
         int size = 0;
         int *intPtr = nullptr;
         double *dataPtr = nullptr;
@@ -98,5 +101,60 @@ namespace bufr {
     {
         delete_table_data_f();
     }
+
+    TypeInfo DataProvider::getTypeInfo(FortranIdx idx) const
+    {
+        static const unsigned int UNIT_STR_LEN = 24;
+        static const unsigned int DESC_STR_LEN = 55;
+
+        char unitCStr[UNIT_STR_LEN];
+        char descCStr[DESC_STR_LEN];
+
+        int retVal;
+        TypeInfo info;
+
+        nemdefs_f(fileUnit_,
+                  getTag(idx).c_str(),
+                   unitCStr,
+                   UNIT_STR_LEN,
+                   descCStr,
+                   DESC_STR_LEN,
+                   &retVal);
+
+        if (retVal == 0)
+        {
+            // trim the unit string
+            auto unitStr = std::string(unitCStr);
+            size_t end = unitStr.find_last_not_of(" \n\r\t\f\v");
+            unitStr = (end == std::string::npos) ? "" : unitStr.substr(0, end + 1);
+            info.unit = unitStr;
+
+            // trim the unit string
+            auto descStr = std::string(descCStr);
+            end = descStr.find_last_not_of(" \n\r\t\f\v");
+            descStr = (end == std::string::npos) ? "" : descStr.substr(0, end + 1);
+            info.description = descStr;
+
+            int descriptor;
+            int table_idx;
+            char table_type;
+
+            nemtab_f(bufrLoc_,
+                     getTag(idx).c_str(),
+                     &descriptor,
+                     &table_type,
+                     &table_idx);
+
+            nemtbb_f(bufrLoc_,
+                     table_idx,
+                     unitCStr,
+                     UNIT_STR_LEN,
+                     &info.scale,
+                     &info.reference,
+                     &info.bits);
+        }
+
+        return info;
+    }
 }  // namespace bufr
 }  // namespace Ingester

src/bufr/BufrParser/Query/DataProvider.h

@@ -9,8 +9,8 @@
 
 #include <string>
 #include <vector>
+#include <math.h>
 #include <memory>
-
 #include <gsl/gsl-lite.hpp>
 
 namespace Ingester{
@@ -31,11 +31,48 @@ namespace bufr {
         Character
     };
 
+    struct TypeInfo
+    {
+        int scale = 0;
+        int reference = 0;
+        int bits = 0;
+        std::string unit;
+        std::string description;
+
+        bool isString() const { return unit == "CCITT IA5"; }
+        bool isSigned() const
+        {
+            // To better support Fortran clients for the generated ObsGroups we will assume all
+            // fields are signed. Otherwise this code would be reference < 0.
+            return true;
+        }
+        bool isInteger() const { return scale <= 0; }
+        bool is64Bit() const
+        {
+            bool is64Bit;
+            if (isInteger() && !isSigned())
+            {
+                is64Bit = (log2((pow(2, bits) - 1) / pow(10, scale) + reference) > 32);
+            }
+            else if (isInteger() && isSigned())
+            {
+                is64Bit = (log2(fmax(-1 * reference,
+                    (pow(2, bits - 1) - 1) / pow(10, scale) + reference) * 2) + 1 > 32);
+            }
+            else
+            {
+                is64Bit = false;
+            }
+
+            return is64Bit;
+        }
+    };
+
     /// \brief Responsible for exposing the data found in a BUFR file in a C friendly way.
     class DataProvider
     {
      public:
-        DataProvider() = default;
+        explicit DataProvider(int fileUnit) : fileUnit_(fileUnit) {}
         ~DataProvider() = default;
 
         /// \brief Read the data from the BUFR interface for the current subset and reset the
@@ -63,8 +100,10 @@ namespace bufr {
         inline FortranIdx getNVal() const { return nval_; }
         inline FortranIdx getInv(FortranIdx idx) const { return inv_[idx - 1]; }
         inline double getVal(FortranIdx idx) const { return val_[idx - 1]; }
+        TypeInfo getTypeInfo(FortranIdx idx) const;
 
      private:
+        int fileUnit_;
         std::string subset_;
 
         // Table data;
@@ -78,6 +117,7 @@ namespace bufr {
         // Subset data
         int inode_;
         int nval_;
+        int bufrLoc_;
         gsl::span<const double> val_;
         gsl::span<const int> inv_;
     };

src/bufr/BufrParser/Query/File.cpp

@@ -67,7 +67,7 @@ namespace bufr {
         int bufrLoc;
         int il, im;  // throw away
 
-        auto dataProvider = DataProvider();
+        auto dataProvider = DataProvider(fileUnit_);
 
         auto resultSet = ResultSet(querySet.names());
         auto query = Query(querySet, resultSet, dataProvider);
@@ -84,6 +84,8 @@ namespace bufr {
             if (next > 0 && ++messageNum >= next) break;
         }
 
+        resultSet.setTargets(query.getTargets());
+
         dataProvider.deleteData();
 
         return resultSet;