Make dateTime fill value consistent across platforms

src/compo/modis_aod2ioda.py

@@ -27,7 +27,7 @@
 locationKeyList = [
     ("latitude", "float"),
     ("longitude", "float"),
-    ("dateTime", "integer")
+    ("dateTime", "long")
 ]
 
 obsvars = ["aerosolOpticalDepth"]
@@ -52,9 +52,9 @@
 
 
 class AOD(object):
-    def __init__(self, filenames, obs_time):
+    def __init__(self, filenames, obs_time_arg):
         self.filenames = filenames
-        self.obs_time = obs_time
+        self.obs_time = obs_time_arg
         self.varDict = defaultdict(lambda: defaultdict(dict))
         self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict))
@@ -79,13 +79,13 @@ def _read(self):
         self.varAttrs[('dateTime', metaDataName)]['units'] = 'seconds since 1993-01-01T00:00:00Z'
 
         # Make empty lists for the output vars
-        self.outdata[('latitude', metaDataName)] = []
-        self.outdata[('longitude', metaDataName)] = []
-        self.outdata[('dateTime', metaDataName)] = []
+        self.outdata[('latitude', metaDataName)] = np.array([], dtype=np.float64)
+        self.outdata[('longitude', metaDataName)] = np.array([], dtype=np.float64)
+        self.outdata[('dateTime', metaDataName)] = np.array([], dtype=np.int64)
         for iodavar in obsvars:
-            self.outdata[self.varDict[iodavar]['valKey']] = []
-            self.outdata[self.varDict[iodavar]['errKey']] = []
-            self.outdata[self.varDict[iodavar]['qcKey']] = []
+            self.outdata[self.varDict[iodavar]['valKey']] = np.array([], dtype=np.float64)
+            self.outdata[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float64)
+            self.outdata[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32)
 
         # loop through input filenames
         for f in self.filenames:

src/conventional/metar_csv2ioda.py

@@ -39,7 +39,7 @@
                    ("longitude", "float", "degrees_east"),
                    ("stationElevation", "float", "m"),
                    ("height", "float", "m"),
-                   ("dateTime", "integer", "seconds since 1970-01-01T00:00:00Z")]
+                   ("dateTime", "long", "seconds since 1970-01-01T00:00:00Z")]
 meta_keys = [m_item[0] for m_item in locationKeyList]
 
 obsvars = ['airTemperature',

src/lib-python/ioda_conv_engines.py

@@ -39,7 +39,7 @@ def OqcName():
     return _oqc_name
 
 
-def get_default_fill_val(mydtype):
+def get_default_fill_val(mydtype, isDateTime=False):
     dtype_tmp = np.array([], dtype=mydtype)
     NumpyDtype = dtype_tmp.dtype
     if (NumpyDtype == np.dtype('float64')):
@@ -59,7 +59,10 @@ def get_default_fill_val(mydtype):
     elif (NumpyDtype == np.dtype('U1')):
         fillval = '\x00'
     elif (NumpyDtype == np.dtype('object')):
-        fillval = '\x00'
+        if (isDateTime):
+            fillval = dt.datetime(2200, 1, 1, tzinfo=dt.timezone.utc)
+        else:
+            fillval = '\x00'
     else:
         print("ERROR: Unrecognized data type", NumpyDtype)
         exit(-2)
@@ -94,7 +97,7 @@ def WriteGeoVars(self, GeoVars, GeoVarDims, GeoVarAttrs):
             else:
                 # assume it is just nlocs
                 dims = ['Location']
-            fillval = get_default_fill_val(Vvals.dtype)
+            fillval = get_default_fill_val(Vvals.dtype, isinstance(Vvals[0], dt.datetime))
             # get fill value
             if VarName in GeoVarAttrs.keys():
                 if '_FillValue' in GeoVarAttrs[VarName].keys():
@@ -123,7 +126,7 @@ def WriteObsVars(self, ObsVars, VarDims, VarAttrs):
             else:
                 # assume it is just nlocs
                 dims = ['Location']
-            fillval = get_default_fill_val(Vvals.dtype)
+            fillval = get_default_fill_val(Vvals.dtype, isinstance(Vvals[0], dt.datetime))
             # get fill value
             if VarKey in VarAttrs.keys():
                 if '_FillValue' in VarAttrs[VarKey].keys():
@@ -199,6 +202,7 @@ def ExtractObsData(ObsData, loc_key_list):
     # can be preallocated, and variable numbers can be assigned
     ObsVarList = []
     ObsVarExamples = []
+    ObsVarTypes = []
     for LocKey, LocDict in ObsData.items():
         _nlocs += 1
         for VarKey, VarVal in LocDict.items():
@@ -207,22 +211,31 @@ def ExtractObsData(ObsData, loc_key_list):
             if (VarKey not in ObsVarList):
                 ObsVarList.append(VarKey)
                 ObsVarExamples.append(VarVal)
+                ObsVarTypes.append(type(VarVal))
         # Extract the locations metadata encoded in the keys
         for i in range(len(loc_key_list)):
             (LocVname, LocVtype) = loc_key_list[i]
             locvar = (LocVname, 'MetaData')
             if (locvar not in ObsVarList):
                 ObsVarList.append(locvar)
                 ObsVarExamples.append(LocKey[i])
+                if (LocVtype == "long"):
+                    # For case where MetaData/dateTime is directly assigned 64-bit integers
+                    ObsVarTypes.append(np.int64)
+                else:
+                    ObsVarTypes.append(type(LocKey[i]))
 
     # Preallocate arrays and fill them up with data from the dictionary
     ObsVars = OrderedDict()
     for o in range(len(ObsVarList)):
-        VarType = type(ObsVarExamples[o])
+        VarType = ObsVarTypes[o]
         if (VarType in [float, np.float32, np.float64]):
             defaultval = get_default_fill_val(np.float32)
             defaultvaltype = np.float32
-        elif (VarType in [int, np.int64, np.int32, np.int8]):
+        elif (VarType in [np.int64]):
+            defaultval = get_default_fill_val(np.int64)
+            defaultvaltype = np.int64
+        elif (VarType in [int, np.int32, np.int8]):
             defaultval = get_default_fill_val(np.int32)
             defaultvaltype = np.int32
         elif (VarType in [str, np.str_]):