Feature/sprint modis aod converters hb

src/compo/modis_aod2ioda.py

@@ -27,26 +27,28 @@
 locationKeyList = [
     ("latitude", "float"),
     ("longitude", "float"),
-    ("datetime", "string")
+    ("dateTime", "integer")
 ]
 
+obsvars = ["aerosolOpticalDepth"]
 
-obsvars = {
-    'A': "aerosol_optical_depth",
-}
+# A dictionary of global attributes.  More filled in further down.
+AttrData = {}
+AttrData['ioda_object_type'] = 'AOD at 550nm'
 
-AttrData = {
-    'converter': os.path.basename(__file__),
-    'nvars': np.int32(len(obsvars)),
-}
+# A dictionary of variable dimensions.
+DimDict = {}
 
+# A dictionary of variable names and their dimensions.
+VarDims = {'aerosolOpticalDepth': ['Location']}
 
-DimDict = {
-}
+# Get the group names we use the most.
+metaDataName = iconv.MetaDataName()
+obsValName = iconv.OvalName()
+obsErrName = iconv.OerrName()
+qcName = iconv.OqcName()
 
-VarDims = {
-    'aerosol_optical_depth': ['nlocs']
-}
+long_missing_value = nc.default_fillvals['i8']
 
 
 class AOD(object):
@@ -60,22 +62,32 @@ def __init__(self, filenames, obs_time):
 
     def _read(self):
         # set up variable names for IODA
-        for ncvar, iodavar in obsvars.items():
-            self.varDict[iodavar]['valKey'] = iodavar, iconv.OvalName()
-            self.varDict[iodavar]['errKey'] = iodavar, iconv.OerrName()
-            self.varDict[iodavar]['qcKey'] = iodavar, iconv.OqcName()
-            self.varAttrs[iodavar, iconv.OvalName()]['coordinates'] = 'longitude latitude'
-            self.varAttrs[iodavar, iconv.OerrName()]['coordinates'] = 'longitude latitude'
-            self.varAttrs[iodavar, iconv.OqcName()]['coordinates'] = 'longitude latitude'
-            self.varAttrs[iodavar, iconv.OvalName()]['_FillValue'] = -9999.
-            self.varAttrs[iodavar, iconv.OerrName()]['_FillValue'] = -9999.
-            self.varAttrs[iodavar, iconv.OqcName()]['_FillValue'] = -9999
-            self.varAttrs[iodavar, iconv.OvalName()]['units'] = '1'
-            self.varAttrs[iodavar, iconv.OqcName()]['units'] = 'unitless'
-            self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'unitless'
+        for iodavar in obsvars:
+            self.varDict[iodavar]['valKey'] = iodavar, obsValName
+            self.varDict[iodavar]['errKey'] = iodavar, obsErrName
+            self.varDict[iodavar]['qcKey'] = iodavar, qcName
+            self.varAttrs[iodavar, obsValName]['coordinates'] = 'longitude latitude'
+            self.varAttrs[iodavar, obsErrName]['coordinates'] = 'longitude latitude'
+            self.varAttrs[iodavar, qcName]['coordinates'] = 'longitude latitude'
+            self.varAttrs[iodavar, obsValName]['_FillValue'] = -9999.
+            self.varAttrs[iodavar, obsErrName]['_FillValue'] = -9999.
+            self.varAttrs[iodavar, qcName]['_FillValue'] = -9999
+            self.varAttrs[iodavar, obsValName]['units'] = '1'
+            self.varAttrs[iodavar, obsErrName]['units'] = '1'
+
+        # All of MODIS AOD data have a singular reference time
+        self.varAttrs[('dateTime', metaDataName)]['units'] = 'seconds since 1993-01-01T00:00:00Z'
+
+        # Make empty lists for the output vars
+        self.outdata[('latitude', metaDataName)] = []
+        self.outdata[('longitude', metaDataName)] = []
+        self.outdata[('dateTime', metaDataName)] = []
+        for iodavar in obsvars:
+            self.outdata[self.varDict[iodavar]['valKey']] = []
+            self.outdata[self.varDict[iodavar]['errKey']] = []
+            self.outdata[self.varDict[iodavar]['qcKey']] = []
 
         # loop through input filenames
-        first = True
         for f in self.filenames:
             ncd = nc.Dataset(f, 'r')
             ncd_str = str(ncd)
@@ -86,14 +98,9 @@ def _read(self):
             if 'MODIS' in ncd_str:
                 AttrData['sensor'] = 'v.modis_terra'
 
-            obstime = self.obs_time
-            AttrData['date_time'] = self.obs_time
-            AttrData['observation_type'] = 'Aod'
-
             #  Get variables
             modis_time = ncd.variables['Scan_Start_Time'][:].ravel()
-
-            # convert time to date_string
+            modis_time = modis_time.astype('float64')
             lats = ncd.variables['Latitude'][:].ravel()
             lats = lats.astype('float32')
             lons = ncd.variables['Longitude'][:].ravel()
@@ -117,10 +124,9 @@ def _read(self):
             sen_zen = sen_zen[pos_index]
             unc_land = unc_land[pos_index]
             modis_time = modis_time[pos_index]
-            obs_time = np.empty_like(QC_flag, dtype=object)
-            obs_time_2 = [datetime.fromisoformat('1993-01-01') + timedelta(seconds=x) for x in modis_time]
-            for t in range(len(obs_time_2)):
-                obs_time[t] = obs_time_2[t].strftime('%Y-%m-%dT%H:%M:%SZ')
+            obs_time = np.full(len(modis_time), long_missing_value, dtype=np.int64)
+            for n, t in enumerate(modis_time):
+                obs_time[n] = round(t)
 
             # uncertainty estimates:
             # From MODIS file (over ocean) and Levy, 2010 (over land)
@@ -129,34 +135,17 @@ def _read(self):
             over_ocean = np.logical_not(land_sea_flag > 0)
             over_land = np.logical_not(land_sea_flag == 0)
             UNC = np.where(over_land, unc_land, np.add(0.05, np.multiply(0.15, aod)))
-            if first:
-                self.outdata[('latitude', 'MetaData')] = lats
-                self.outdata[('longitude', 'MetaData')] = lons
-                self.outdata[('datetime', 'MetaData')] = obs_time
-            else:
-                self.outdata[('latitude', 'MetaData')] = np.concatenate((self.outdata[('latitude', 'MetaData')], lats))
-                self.outdata[('longitude', 'MetaData')] = np.concatenate((self.outdata[('longitude', 'MetaData')], lons))
-                self.outdata[('datetime', 'MetaData')] = np.concatenate((self.outdata[('datetime', 'MetaData')], obs_time))
-
-            for ncvar, iodavar in obsvars.items():
-                data = aod.astype('float32')
-                err = UNC.astype('float32')
-                if first:
-                    self.outdata[self.varDict[iodavar]['valKey']] = data
-                    self.outdata[self.varDict[iodavar]['errKey']] = err
-                    self.outdata[self.varDict[iodavar]['qcKey']] = QC_flag
-
-                else:
-                    self.outdata[self.varDict[iodavar]['valKey']] = np.concatenate(
-                        (self.outdata[self.varDict[iodavar]['valKey']], data))
-                    self.outdata[self.varDict[iodavar]['errKey']] = np.concatenate(
-                        (self.outdata[self.varDict[iodavar]['errKey']], err))
-                    self.outdata[self.varDict[iodavar]['qcKey']] = np.concatenate(
-                        (self.outdata[self.varDict[iodavar]['qcKey']], QC_flag))
-
-            first = False
-        DimDict['nlocs'] = len(self.outdata[('datetime', 'MetaData')])
-        AttrData['nlocs'] = np.int32(DimDict['nlocs'])
+
+            self.outdata[('latitude', metaDataName)] = np.append(self.outdata[('latitude', metaDataName)], np.array(lats, dtype=np.float32))
+            self.outdata[('longitude', metaDataName)] = np.append(self.outdata[('longitude', metaDataName)], np.array(lons, dtype=np.float32))
+            self.outdata[('dateTime', metaDataName)] = np.append(self.outdata[('dateTime', metaDataName)], np.array(obs_time, dtype=np.int64))
+
+            for iodavar in obsvars:
+                self.outdata[self.varDict[iodavar]['valKey']] = np.append(self.outdata[self.varDict[iodavar]['valKey']], np.array(aod, dtype=np.float32))
+                self.outdata[self.varDict[iodavar]['errKey']] = np.append(self.outdata[self.varDict[iodavar]['errKey']], np.array(UNC, dtype=np.float32))
+                self.outdata[self.varDict[iodavar]['qcKey']] = np.append(self.outdata[self.varDict[iodavar]['qcKey']], np.array(QC_flag, dtype=np.int32))
+
+        DimDict['Location'] = len(self.outdata[('dateTime', metaDataName)])
 
 
 def main():