diff --git a/src/compo/modis_aod2ioda.py b/src/compo/modis_aod2ioda.py index 8eca2d48e..015e8f801 100644 --- a/src/compo/modis_aod2ioda.py +++ b/src/compo/modis_aod2ioda.py @@ -27,26 +27,28 @@ locationKeyList = [ ("latitude", "float"), ("longitude", "float"), - ("datetime", "string") + ("dateTime", "integer") ] +obsvars = ["aerosolOpticalDepth"] -obsvars = { - 'A': "aerosol_optical_depth", -} +# A dictionary of global attributes. More filled in further down. +AttrData = {} +AttrData['ioda_object_type'] = 'AOD at 550nm' -AttrData = { - 'converter': os.path.basename(__file__), - 'nvars': np.int32(len(obsvars)), -} +# A dictionary of variable dimensions. +DimDict = {} +# A dictionary of variable names and their dimensions. +VarDims = {'aerosolOpticalDepth': ['Location']} -DimDict = { -} +# Get the group names we use the most. +metaDataName = iconv.MetaDataName() +obsValName = iconv.OvalName() +obsErrName = iconv.OerrName() +qcName = iconv.OqcName() -VarDims = { - 'aerosol_optical_depth': ['nlocs'] -} +long_missing_value = nc.default_fillvals['i8'] class AOD(object): @@ -60,22 +62,32 @@ def __init__(self, filenames, obs_time): def _read(self): # set up variable names for IODA - for ncvar, iodavar in obsvars.items(): - self.varDict[iodavar]['valKey'] = iodavar, iconv.OvalName() - self.varDict[iodavar]['errKey'] = iodavar, iconv.OerrName() - self.varDict[iodavar]['qcKey'] = iodavar, iconv.OqcName() - self.varAttrs[iodavar, iconv.OvalName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OerrName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OqcName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OvalName()]['_FillValue'] = -9999. - self.varAttrs[iodavar, iconv.OerrName()]['_FillValue'] = -9999. - self.varAttrs[iodavar, iconv.OqcName()]['_FillValue'] = -9999 - self.varAttrs[iodavar, iconv.OvalName()]['units'] = '1' - self.varAttrs[iodavar, iconv.OqcName()]['units'] = 'unitless' - self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'unitless' + for iodavar in obsvars: + self.varDict[iodavar]['valKey'] = iodavar, obsValName + self.varDict[iodavar]['errKey'] = iodavar, obsErrName + self.varDict[iodavar]['qcKey'] = iodavar, qcName + self.varAttrs[iodavar, obsValName]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, obsErrName]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, qcName]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, obsValName]['_FillValue'] = -9999. + self.varAttrs[iodavar, obsErrName]['_FillValue'] = -9999. + self.varAttrs[iodavar, qcName]['_FillValue'] = -9999 + self.varAttrs[iodavar, obsValName]['units'] = '1' + self.varAttrs[iodavar, obsErrName]['units'] = '1' + + # All of MODIS AOD data have a singular reference time + self.varAttrs[('dateTime', metaDataName)]['units'] = 'seconds since 1993-01-01T00:00:00Z' + + # Make empty lists for the output vars + self.outdata[('latitude', metaDataName)] = [] + self.outdata[('longitude', metaDataName)] = [] + self.outdata[('dateTime', metaDataName)] = [] + for iodavar in obsvars: + self.outdata[self.varDict[iodavar]['valKey']] = [] + self.outdata[self.varDict[iodavar]['errKey']] = [] + self.outdata[self.varDict[iodavar]['qcKey']] = [] # loop through input filenames - first = True for f in self.filenames: ncd = nc.Dataset(f, 'r') ncd_str = str(ncd) @@ -86,14 +98,9 @@ def _read(self): if 'MODIS' in ncd_str: AttrData['sensor'] = 'v.modis_terra' - obstime = self.obs_time - AttrData['date_time'] = self.obs_time - AttrData['observation_type'] = 'Aod' - # Get variables modis_time = ncd.variables['Scan_Start_Time'][:].ravel() - - # convert time to date_string + modis_time = modis_time.astype('float64') lats = ncd.variables['Latitude'][:].ravel() lats = lats.astype('float32') lons = ncd.variables['Longitude'][:].ravel() @@ -117,10 +124,9 @@ def _read(self): sen_zen = sen_zen[pos_index] unc_land = unc_land[pos_index] modis_time = modis_time[pos_index] - obs_time = np.empty_like(QC_flag, dtype=object) - obs_time_2 = [datetime.fromisoformat('1993-01-01') + timedelta(seconds=x) for x in modis_time] - for t in range(len(obs_time_2)): - obs_time[t] = obs_time_2[t].strftime('%Y-%m-%dT%H:%M:%SZ') + obs_time = np.full(len(modis_time), long_missing_value, dtype=np.int64) + for n, t in enumerate(modis_time): + obs_time[n] = round(t) # uncertainty estimates: # From MODIS file (over ocean) and Levy, 2010 (over land) @@ -129,34 +135,17 @@ def _read(self): over_ocean = np.logical_not(land_sea_flag > 0) over_land = np.logical_not(land_sea_flag == 0) UNC = np.where(over_land, unc_land, np.add(0.05, np.multiply(0.15, aod))) - if first: - self.outdata[('latitude', 'MetaData')] = lats - self.outdata[('longitude', 'MetaData')] = lons - self.outdata[('datetime', 'MetaData')] = obs_time - else: - self.outdata[('latitude', 'MetaData')] = np.concatenate((self.outdata[('latitude', 'MetaData')], lats)) - self.outdata[('longitude', 'MetaData')] = np.concatenate((self.outdata[('longitude', 'MetaData')], lons)) - self.outdata[('datetime', 'MetaData')] = np.concatenate((self.outdata[('datetime', 'MetaData')], obs_time)) - - for ncvar, iodavar in obsvars.items(): - data = aod.astype('float32') - err = UNC.astype('float32') - if first: - self.outdata[self.varDict[iodavar]['valKey']] = data - self.outdata[self.varDict[iodavar]['errKey']] = err - self.outdata[self.varDict[iodavar]['qcKey']] = QC_flag - - else: - self.outdata[self.varDict[iodavar]['valKey']] = np.concatenate( - (self.outdata[self.varDict[iodavar]['valKey']], data)) - self.outdata[self.varDict[iodavar]['errKey']] = np.concatenate( - (self.outdata[self.varDict[iodavar]['errKey']], err)) - self.outdata[self.varDict[iodavar]['qcKey']] = np.concatenate( - (self.outdata[self.varDict[iodavar]['qcKey']], QC_flag)) - - first = False - DimDict['nlocs'] = len(self.outdata[('datetime', 'MetaData')]) - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + + self.outdata[('latitude', metaDataName)] = np.append(self.outdata[('latitude', metaDataName)], np.array(lats, dtype=np.float32)) + self.outdata[('longitude', metaDataName)] = np.append(self.outdata[('longitude', metaDataName)], np.array(lons, dtype=np.float32)) + self.outdata[('dateTime', metaDataName)] = np.append(self.outdata[('dateTime', metaDataName)], np.array(obs_time, dtype=np.int64)) + + for iodavar in obsvars: + self.outdata[self.varDict[iodavar]['valKey']] = np.append(self.outdata[self.varDict[iodavar]['valKey']], np.array(aod, dtype=np.float32)) + self.outdata[self.varDict[iodavar]['errKey']] = np.append(self.outdata[self.varDict[iodavar]['errKey']], np.array(UNC, dtype=np.float32)) + self.outdata[self.varDict[iodavar]['qcKey']] = np.append(self.outdata[self.varDict[iodavar]['qcKey']], np.array(QC_flag, dtype=np.int32)) + + DimDict['Location'] = len(self.outdata[('dateTime', metaDataName)]) def main(): diff --git a/test/testoutput/modis_aod.nc b/test/testoutput/modis_aod.nc index 46960d731..9f8eb13e1 100644 --- a/test/testoutput/modis_aod.nc +++ b/test/testoutput/modis_aod.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f0192dbc1f675e035d2beb40db4256324096bb4a438db29ebe0d418b59ead75 -size 34699 +oid sha256:0622a99d75d1cd0a280e6c86ac3cd1b4d601e2b504c7b4d94d85bff9e936a72d +size 18923