Feature/query datetime subset fix

src/bufr/BufrParser/BufrParser.cpp

@@ -48,7 +48,8 @@ namespace Ingester {
     {
         auto startTime = std::chrono::steady_clock::now();
 
-        auto querySet = bufr::QuerySet();
+        auto querySet = bufr::QuerySet(description_.getExport().getSubsets());
+
         for (const auto &var : description_.getExport().getVariables())
         {
             for (const auto &queryPair : var->getQueryList())

src/bufr/BufrParser/Exports/Export.cpp

@@ -28,6 +28,7 @@ namespace
         const char* Splits = "splits";
         const char* Variables = "variables";
         const char* GroupByVariable = "group_by_variable";
+        const char* Subsets = "subsets";
 
         namespace Variable
         {
@@ -75,6 +76,11 @@ namespace Ingester
             groupByVariable = conf.getString(ConfKeys::GroupByVariable);
         }
 
+        if (conf.has(ConfKeys::Subsets))
+        {
+            subsets_ = conf.getStringVector(ConfKeys::Subsets);
+        }
+
         if (conf.has(ConfKeys::Variables))
         {
             addVariables(conf.getSubConfiguration(ConfKeys::Variables),

src/bufr/BufrParser/Exports/Export.h

@@ -35,11 +35,13 @@ namespace Ingester
         inline Splits getSplits() const { return splits_; }
         inline Variables getVariables() const { return variables_; }
         inline Filters getFilters() const { return filters_; }
+        inline std::vector<std::string> getSubsets() const { return subsets_; }
 
      private:
         Splits splits_;
         Variables  variables_;
         Filters filters_;
+        std::vector<std::string> subsets_;
 
         /// \brief Create Variables exports from config.
         void addVariables(const eckit::Configuration &conf,

src/bufr/BufrParser/Exports/Splits/CategorySplit.cpp

@@ -83,11 +83,10 @@ namespace Ingester
                 auto location = Location(dataObject->getDims().size(), 0);
                 location[0] = rowIdx;
 
-                auto itemVal =  dataObject->getAsFloat(location);
-                if (trunc(itemVal) == itemVal)
+                if (auto dat = std::dynamic_pointer_cast<DataObject<int>> (dataObject))
                 {
-                    nameMap_.insert({static_cast<int> (itemVal),
-                                     std::to_string(static_cast<int> (itemVal))});
+                    auto itemVal = dat->get(location);
+                    nameMap_.insert({itemVal, std::to_string(itemVal)});
                 }
                 else
                 {

src/bufr/BufrParser/Exports/Variables/DatetimeVariable.cpp

@@ -8,6 +8,7 @@
 #include <climits>
 #include <iostream>
 #include <iomanip>
+#include <unordered_map>
 
 #include <ostream>
 #include <time.h>
@@ -71,8 +72,7 @@ namespace Ingester
     std::shared_ptr<DataObjectBase> DatetimeVariable::exportData(const BufrDataMap& map)
     {
         checkKeys(map);
-        static const float missing = 1.e+11;
-        static const int64_t missing_int = INT_MIN;
+        static const int missingInt = DataObject<int>::missingValue();
 
         std::tm tm{};                // zero initialise
         tm.tm_year = 1970-1900;      // 1970
@@ -83,26 +83,38 @@ namespace Ingester
         tm.tm_sec = 0;
         tm.tm_isdst = 0;             // Not daylight saving
         std::time_t epochDt = std::mktime(&tm);
-        std::time_t this_time = std::mktime(&tm);
-        int64_t diff_time;
 
         std::vector<int64_t> timeOffsets;
         timeOffsets.reserve(map.at(getExportKey(ConfKeys::Year))->size());
 
+        // Validation
+        if (map.at(getExportKey(ConfKeys::Year))->getDims().size() != 1 ||
+            map.at(getExportKey(ConfKeys::Month))->getDims().size() != 1 ||
+            map.at(getExportKey(ConfKeys::Day))->getDims().size() != 1 ||
+            (!minuteQuery_.empty() &&
+                map.at(getExportKey(ConfKeys::Minute))->getDims().size() != 1) ||
+            (!secondQuery_.empty() &&
+                map.at(getExportKey(ConfKeys::Second))->getDims().size() != 1))
+        {
+            std::ostringstream errStr;
+            errStr << "Datetime variables must be 1 dimensional.";
+            throw eckit::BadParameter(errStr.str());
+        }
+
         for (unsigned int idx = 0; idx < map.at(getExportKey(ConfKeys::Year))->size(); idx++)
         {
-            int year = static_cast<int>(map.at(getExportKey(ConfKeys::Year))->getAsFloat(idx));
-            int month = static_cast<int>(map.at(getExportKey(ConfKeys::Month))->getAsFloat(idx));
-            int day = static_cast<int>(map.at(getExportKey(ConfKeys::Day))->getAsFloat(idx));
-            int hour = static_cast<int>(map.at(getExportKey(ConfKeys::Hour))->getAsFloat(idx));
+            int year = map.at(getExportKey(ConfKeys::Year))->getAsInt(idx);
+            int month = map.at(getExportKey(ConfKeys::Month))->getAsInt(idx);
+            int day = map.at(getExportKey(ConfKeys::Day))->getAsInt(idx);
+            int hour = map.at(getExportKey(ConfKeys::Hour))->getAsInt(idx);
             int minutes = 0;
             int seconds = 0;
 
-            diff_time = missing_int;
-            if (year != missing &&
-                month != missing &&
-                day != missing &&
-                hour != missing)
+            auto diff_time = DataObject<int64_t>::missingValue();
+            if (year != missingInt &&
+                month != missingInt &&
+                day != missingInt &&
+                hour != missingInt)
             {
                 tm.tm_year = year - 1900;
                 tm.tm_mon = month - 1;
@@ -114,8 +126,7 @@ namespace Ingester
 
                 if (!minuteQuery_.empty())
                 {
-                    minutes =
-                        static_cast<int>(map.at(getExportKey(ConfKeys::Minute))->getAsFloat(idx));
+                    minutes = map.at(getExportKey(ConfKeys::Minute))->getAsInt(idx);
 
                     if (minutes >= 0 && minutes < 60)
                     {
@@ -125,25 +136,26 @@ namespace Ingester
 
                 if (!secondQuery_.empty())
                 {
-                    seconds =
-                        static_cast<int>(map.at(getExportKey(ConfKeys::Second))->getAsFloat(idx));
+                    seconds = map.at(getExportKey(ConfKeys::Second))->getAsInt(idx);
 
                     if (seconds >= 0 && seconds < 60)
                     {
                         tm.tm_sec = seconds;
                     }
                 }
 
-                this_time = std::mktime(&tm);
-                if (this_time < 0)
+                // Be careful with mktime as it can be very slow.
+                auto thisTime = std::mktime(&tm);
+                if (thisTime < 0)
                 {
                      oops::Log::warning() << "Caution, date suspicious date (year, month, day): "
                                           << year << ", "
                                           << month << ", "
                                           << day << std::endl;
                 }
-                diff_time = static_cast<std::int64_t>(difftime(this_time, epochDt)
-                                                      + hoursFromUtc_*3600);
+
+                diff_time = static_cast<int64_t>(difftime(thisTime, epochDt)
+                                                 + hoursFromUtc_ * 3600);
             }
 
             timeOffsets.push_back(diff_time);

src/bufr/BufrParser/Query/File.cpp

@@ -7,9 +7,11 @@
 
 #include "File.h"
 
+#include <algorithm>
+
 #include "bufr_interface.h"
 
-#include "Query.h"
+#include "QueryRunner.h"
 #include "QuerySet.h"
 #include "DataProvider.h"
 
@@ -61,7 +63,7 @@ namespace bufr {
     {
         static int SubsetLen = 9;
         unsigned int messageNum = 0;
-        char subset[SubsetLen];
+        char subsetChars[SubsetLen];
         int iddate;
 
         int bufrLoc;
@@ -70,21 +72,27 @@ namespace bufr {
         auto dataProvider = DataProvider(fileUnit_);
 
         auto resultSet = ResultSet(querySet.names());
-        auto query = Query(querySet, resultSet, dataProvider);
+        auto queryRunner = QueryRunner(querySet, resultSet, dataProvider);
 
-        while (ireadmg_f(fileUnit_, subset, &iddate, SubsetLen) == 0)
+        while (ireadmg_f(fileUnit_, subsetChars, &iddate, SubsetLen) == 0)
         {
-            while (ireadsb_f(fileUnit_) == 0)
+            auto subset = std::string(subsetChars);
+            subset.erase(std::remove_if(subset.begin(), subset.end(), isspace), subset.end());
+
+            if (querySet.includesSubset(subset))
             {
-                status_f(fileUnit_, &bufrLoc, &il, &im);
-                dataProvider.updateData(bufrLoc);
-                query.query();
+                while (ireadsb_f(fileUnit_) == 0)
+                {
+                    status_f(fileUnit_, &bufrLoc, &il, &im);
+                    dataProvider.updateData(bufrLoc);
+                    queryRunner.accumulate();
+                }
+
+                if (next > 0 && ++messageNum >= next) break;
             }
-
-            if (next > 0 && ++messageNum >= next) break;
         }
 
-        resultSet.setTargets(query.getTargets());
+        resultSet.setTargets(queryRunner.getTargets());
 
         dataProvider.deleteData();
 

src/bufr/BufrParser/Query/QueryParser.cpp

@@ -14,7 +14,19 @@
 
 namespace Ingester {
 namespace bufr {
-     std::vector<std::string> QueryParser::splitMultiquery(const std::string &query) {
+    std::vector<Query> QueryParser::parse(const std::string& queryStr)
+    {
+        std::vector<Query> queries;
+        for (auto& subStr : QueryParser::splitMultiquery(queryStr))
+        {
+            queries.emplace_back(QueryParser::splitQueryStr(subStr));
+        }
+
+        return queries;
+    }
+
+    std::vector<std::string> QueryParser::splitMultiquery(const std::string &query)
+    {
         std::vector<std::string> subqueries;
 
         // Remove whitespace from query and assign to working_str
@@ -66,11 +78,8 @@ namespace bufr {
         return subqueries;
     }
 
-
-    void QueryParser::splitQueryStr(const std::string& query,
-                                    std::string& subset,
-                                    std::vector<std::string>& mnemonics,
-                                    int& index) {
+    Query QueryParser::splitQueryStr(const std::string& query)
+    {
         // Find positions of slashes
         std::vector<size_t> slashPositions;
         size_t slashIdx = 0;
@@ -89,7 +98,7 @@ namespace bufr {
         }
 
         // Capture the subset string
-        subset = query.substr(0, slashPositions[0]);
+        auto subset = query.substr(0, slashPositions[0]);
 
         std::vector<std::string> mnemonicStrings(slashPositions.size());
 
@@ -105,7 +114,7 @@ namespace bufr {
         std::string lastElement = query.substr(slashPositions[slashPositions.size() - 1] + 1);
 
         // Parse last element
-        index = -1;
+        int index = -1;
         size_t startSubscript = lastElement.find_first_of("[");
         size_t endSubscript = lastElement.find_first_of("]");
         if (startSubscript != std::string::npos && endSubscript != std::string::npos)
@@ -126,7 +135,13 @@ namespace bufr {
             mnemonicStrings.back() = lastElement;
         }
 
-        mnemonics = mnemonicStrings;
+        auto queryObj = Query();
+        queryObj.queryStr = query;
+        queryObj.subset = subset;
+        queryObj.mnemonics = mnemonicStrings;
+        queryObj.index = index;
+
+        return queryObj;
     }
 }  // namespace bufr
 }  // namespace Ingester

src/bufr/BufrParser/Query/QueryParser.h

@@ -13,24 +13,29 @@
 namespace Ingester {
 namespace bufr {
 
+    struct Query
+    {
+        std::string queryStr;
+        std::string subset;
+        std::vector<std::string> mnemonics;
+        int index;
+    };
+
     /// \brief Parses a user supplied query string into its component parts.
     /// \note Will be refactored to properly tokenize the query string.
     class QueryParser
     {
      public:
+        static std::vector<Query> parse(const std::string& queryStr);
+
+     private:
         /// \brief Split a multi query (ex: ["*/CLONH", "*/CLON"]) into a vector of single queries.
         /// \param query The query to split.
         static std::vector<std::string> splitMultiquery(const std::string& query);
 
         /// \brief Split a single query (ex: "*/ROSEQ1/ROSEQ2/PCCF[2]") into its component parts.
         /// \param query The query to split.
-        /// \param[out] subset The subset part of the query (ex: *).
-        /// \param[out] mnemonics  Query path components (ex: ["ROSEQ1", "ROSEQ2", "PCCF"]).
-        /// \param[out] index The index associated with this query (ex: 2).
-        static void splitQueryStr(const std::string& query,
-                                  std::string& subset,
-                                  std::vector<std::string>& mnemonics,
-                                  int& index);
+        static Query splitQueryStr(const std::string& query);
 
      private:
         /// \brief Private constructor.