From fa09220ed5d6a3c077ad2648123c11ba048fb31e Mon Sep 17 00:00:00 2001 From: bhuang95 Date: Wed, 9 Mar 2022 22:49:22 +0000 Subject: [PATCH 1/6] Changes regarding iodaconv --- src/compo/modis_aod2ioda.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/compo/modis_aod2ioda.py b/src/compo/modis_aod2ioda.py index 8eca2d48e..c86af1b46 100644 --- a/src/compo/modis_aod2ioda.py +++ b/src/compo/modis_aod2ioda.py @@ -15,7 +15,7 @@ import os from pathlib import Path -IODA_CONV_PATH = Path(__file__).parent/"@SCRIPT_LIB_PATH@" +IODA_CONV_PATH = Path(__file__).parent/"../lib/pyiodaconv" if not IODA_CONV_PATH.is_dir(): IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' sys.path.append(str(IODA_CONV_PATH.resolve())) @@ -27,25 +27,25 @@ locationKeyList = [ ("latitude", "float"), ("longitude", "float"), - ("datetime", "string") + ("dateTime", "string") ] obsvars = { - 'A': "aerosol_optical_depth", + 'A': "aerosolOpticalDepth", } AttrData = { - 'converter': os.path.basename(__file__), - 'nvars': np.int32(len(obsvars)), } + #'converter': os.path.basename(__file__), + #'nvars': np.int32(len(obsvars)), DimDict = { } VarDims = { - 'aerosol_optical_depth': ['nlocs'] + 'aerosolOpticalDepth': ['Location', "Channel"] } @@ -71,8 +71,8 @@ def _read(self): self.varAttrs[iodavar, iconv.OerrName()]['_FillValue'] = -9999. self.varAttrs[iodavar, iconv.OqcName()]['_FillValue'] = -9999 self.varAttrs[iodavar, iconv.OvalName()]['units'] = '1' - self.varAttrs[iodavar, iconv.OqcName()]['units'] = 'unitless' - self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'unitless' + self.varAttrs[iodavar, iconv.OerrName()]['units'] = '1' + self.varAttrs[iodavar, iconv.OqcName()]['units'] = '1' # loop through input filenames first = True @@ -86,9 +86,9 @@ def _read(self): if 'MODIS' in ncd_str: AttrData['sensor'] = 'v.modis_terra' - obstime = self.obs_time - AttrData['date_time'] = self.obs_time - AttrData['observation_type'] = 'Aod' + #obstime = self.obs_time + #AttrData['date_time'] = self.obs_time + AttrData['ioda_object_type'] = 'AOD@550nm' # Get variables modis_time = ncd.variables['Scan_Start_Time'][:].ravel() @@ -132,11 +132,11 @@ def _read(self): if first: self.outdata[('latitude', 'MetaData')] = lats self.outdata[('longitude', 'MetaData')] = lons - self.outdata[('datetime', 'MetaData')] = obs_time + self.outdata[('dateTime', 'MetaData')] = obs_time else: self.outdata[('latitude', 'MetaData')] = np.concatenate((self.outdata[('latitude', 'MetaData')], lats)) self.outdata[('longitude', 'MetaData')] = np.concatenate((self.outdata[('longitude', 'MetaData')], lons)) - self.outdata[('datetime', 'MetaData')] = np.concatenate((self.outdata[('datetime', 'MetaData')], obs_time)) + self.outdata[('dateTime', 'MetaData')] = np.concatenate((self.outdata[('dateTime', 'MetaData')], obs_time)) for ncvar, iodavar in obsvars.items(): data = aod.astype('float32') @@ -155,8 +155,10 @@ def _read(self): (self.outdata[self.varDict[iodavar]['qcKey']], QC_flag)) first = False - DimDict['nlocs'] = len(self.outdata[('datetime', 'MetaData')]) - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + self.varAttrs[('dateTime', 'MetaData')]['units'] = '' + DimDict['Location'] = len(self.outdata[('dateTime', 'MetaData')]) + #AttrData['Location'] = np.int32(DimDict['Location']) + DimDict['Channel'] = 2 def main(): From eb74fc6d42015f4df5cddadb0850615f16b00791 Mon Sep 17 00:00:00 2001 From: bhuang95 Date: Thu, 10 Mar 2022 00:54:56 +0000 Subject: [PATCH 2/6] Update chem/modis_aod for JEDI IODA convention --- src/compo/modis_aod2ioda.py | 16 +++++++--------- test/testoutput/modis_aod.nc | 4 ++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/compo/modis_aod2ioda.py b/src/compo/modis_aod2ioda.py index c86af1b46..aee9e0203 100644 --- a/src/compo/modis_aod2ioda.py +++ b/src/compo/modis_aod2ioda.py @@ -15,7 +15,7 @@ import os from pathlib import Path -IODA_CONV_PATH = Path(__file__).parent/"../lib/pyiodaconv" +IODA_CONV_PATH = Path(__file__).parent/"@SCRIPT_LIB_PATH@" if not IODA_CONV_PATH.is_dir(): IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' sys.path.append(str(IODA_CONV_PATH.resolve())) @@ -37,15 +37,13 @@ AttrData = { } - #'converter': os.path.basename(__file__), - #'nvars': np.int32(len(obsvars)), DimDict = { } VarDims = { - 'aerosolOpticalDepth': ['Location', "Channel"] + 'aerosolOpticalDepth': ['Location', 'Channel'] } @@ -86,8 +84,6 @@ def _read(self): if 'MODIS' in ncd_str: AttrData['sensor'] = 'v.modis_terra' - #obstime = self.obs_time - #AttrData['date_time'] = self.obs_time AttrData['ioda_object_type'] = 'AOD@550nm' # Get variables @@ -141,8 +137,11 @@ def _read(self): for ncvar, iodavar in obsvars.items(): data = aod.astype('float32') err = UNC.astype('float32') + data2d = np.array(data).reshape(len(data), 1) + err2d = np.array(err).reshape(len(err), 1) + qc2d = np.array(QC_flag).reshape(len(QC_flag), 1) if first: - self.outdata[self.varDict[iodavar]['valKey']] = data + self.outdata[self.varDict[iodavar]['valKey']] = data2d self.outdata[self.varDict[iodavar]['errKey']] = err self.outdata[self.varDict[iodavar]['qcKey']] = QC_flag @@ -157,8 +156,7 @@ def _read(self): first = False self.varAttrs[('dateTime', 'MetaData')]['units'] = '' DimDict['Location'] = len(self.outdata[('dateTime', 'MetaData')]) - #AttrData['Location'] = np.int32(DimDict['Location']) - DimDict['Channel'] = 2 + DimDict['Channel'] = np.array([2], dtype=np.intc) def main(): diff --git a/test/testoutput/modis_aod.nc b/test/testoutput/modis_aod.nc index 46960d731..e8d918f63 100644 --- a/test/testoutput/modis_aod.nc +++ b/test/testoutput/modis_aod.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6f0192dbc1f675e035d2beb40db4256324096bb4a438db29ebe0d418b59ead75 -size 34699 +oid sha256:640067452f05441b10e8354fcb33f3deeafb340cc92cff6a2de2c30c2a73a4ab +size 32350 From ef2e1e7a4bd18c254fcd45c30d8a74a3927704f2 Mon Sep 17 00:00:00 2001 From: bhuang95 Date: Thu, 10 Mar 2022 22:28:49 +0000 Subject: [PATCH 3/6] Resolove PreQC unit issues --- src/compo/modis_aod2ioda.py | 1 - test/testoutput/modis_aod.nc | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/compo/modis_aod2ioda.py b/src/compo/modis_aod2ioda.py index aee9e0203..f737d044a 100644 --- a/src/compo/modis_aod2ioda.py +++ b/src/compo/modis_aod2ioda.py @@ -70,7 +70,6 @@ def _read(self): self.varAttrs[iodavar, iconv.OqcName()]['_FillValue'] = -9999 self.varAttrs[iodavar, iconv.OvalName()]['units'] = '1' self.varAttrs[iodavar, iconv.OerrName()]['units'] = '1' - self.varAttrs[iodavar, iconv.OqcName()]['units'] = '1' # loop through input filenames first = True diff --git a/test/testoutput/modis_aod.nc b/test/testoutput/modis_aod.nc index e8d918f63..e56b28215 100644 --- a/test/testoutput/modis_aod.nc +++ b/test/testoutput/modis_aod.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:640067452f05441b10e8354fcb33f3deeafb340cc92cff6a2de2c30c2a73a4ab -size 32350 +oid sha256:60ba78c59d21f80d8f52d4bdb465d0543d8cfc600541b0a8bb7ee96301475891 +size 34398 From bf80c13a093031dea512db6811d315e197d647cd Mon Sep 17 00:00:00 2001 From: gthompsnJCSDA Date: Fri, 11 Mar 2022 09:26:25 -0600 Subject: [PATCH 4/6] significant changes, cleaning up, etc. --- src/compo/modis_aod2ioda.py | 119 ++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 65 deletions(-) diff --git a/src/compo/modis_aod2ioda.py b/src/compo/modis_aod2ioda.py index aee9e0203..015e8f801 100644 --- a/src/compo/modis_aod2ioda.py +++ b/src/compo/modis_aod2ioda.py @@ -27,24 +27,28 @@ locationKeyList = [ ("latitude", "float"), ("longitude", "float"), - ("dateTime", "string") + ("dateTime", "integer") ] +obsvars = ["aerosolOpticalDepth"] -obsvars = { - 'A': "aerosolOpticalDepth", -} +# A dictionary of global attributes. More filled in further down. +AttrData = {} +AttrData['ioda_object_type'] = 'AOD at 550nm' -AttrData = { -} +# A dictionary of variable dimensions. +DimDict = {} +# A dictionary of variable names and their dimensions. +VarDims = {'aerosolOpticalDepth': ['Location']} -DimDict = { -} +# Get the group names we use the most. +metaDataName = iconv.MetaDataName() +obsValName = iconv.OvalName() +obsErrName = iconv.OerrName() +qcName = iconv.OqcName() -VarDims = { - 'aerosolOpticalDepth': ['Location', 'Channel'] -} +long_missing_value = nc.default_fillvals['i8'] class AOD(object): @@ -58,22 +62,32 @@ def __init__(self, filenames, obs_time): def _read(self): # set up variable names for IODA - for ncvar, iodavar in obsvars.items(): - self.varDict[iodavar]['valKey'] = iodavar, iconv.OvalName() - self.varDict[iodavar]['errKey'] = iodavar, iconv.OerrName() - self.varDict[iodavar]['qcKey'] = iodavar, iconv.OqcName() - self.varAttrs[iodavar, iconv.OvalName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OerrName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OqcName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OvalName()]['_FillValue'] = -9999. - self.varAttrs[iodavar, iconv.OerrName()]['_FillValue'] = -9999. - self.varAttrs[iodavar, iconv.OqcName()]['_FillValue'] = -9999 - self.varAttrs[iodavar, iconv.OvalName()]['units'] = '1' - self.varAttrs[iodavar, iconv.OerrName()]['units'] = '1' - self.varAttrs[iodavar, iconv.OqcName()]['units'] = '1' + for iodavar in obsvars: + self.varDict[iodavar]['valKey'] = iodavar, obsValName + self.varDict[iodavar]['errKey'] = iodavar, obsErrName + self.varDict[iodavar]['qcKey'] = iodavar, qcName + self.varAttrs[iodavar, obsValName]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, obsErrName]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, qcName]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, obsValName]['_FillValue'] = -9999. + self.varAttrs[iodavar, obsErrName]['_FillValue'] = -9999. + self.varAttrs[iodavar, qcName]['_FillValue'] = -9999 + self.varAttrs[iodavar, obsValName]['units'] = '1' + self.varAttrs[iodavar, obsErrName]['units'] = '1' + + # All of MODIS AOD data have a singular reference time + self.varAttrs[('dateTime', metaDataName)]['units'] = 'seconds since 1993-01-01T00:00:00Z' + + # Make empty lists for the output vars + self.outdata[('latitude', metaDataName)] = [] + self.outdata[('longitude', metaDataName)] = [] + self.outdata[('dateTime', metaDataName)] = [] + for iodavar in obsvars: + self.outdata[self.varDict[iodavar]['valKey']] = [] + self.outdata[self.varDict[iodavar]['errKey']] = [] + self.outdata[self.varDict[iodavar]['qcKey']] = [] # loop through input filenames - first = True for f in self.filenames: ncd = nc.Dataset(f, 'r') ncd_str = str(ncd) @@ -84,12 +98,9 @@ def _read(self): if 'MODIS' in ncd_str: AttrData['sensor'] = 'v.modis_terra' - AttrData['ioda_object_type'] = 'AOD@550nm' - # Get variables modis_time = ncd.variables['Scan_Start_Time'][:].ravel() - - # convert time to date_string + modis_time = modis_time.astype('float64') lats = ncd.variables['Latitude'][:].ravel() lats = lats.astype('float32') lons = ncd.variables['Longitude'][:].ravel() @@ -113,10 +124,9 @@ def _read(self): sen_zen = sen_zen[pos_index] unc_land = unc_land[pos_index] modis_time = modis_time[pos_index] - obs_time = np.empty_like(QC_flag, dtype=object) - obs_time_2 = [datetime.fromisoformat('1993-01-01') + timedelta(seconds=x) for x in modis_time] - for t in range(len(obs_time_2)): - obs_time[t] = obs_time_2[t].strftime('%Y-%m-%dT%H:%M:%SZ') + obs_time = np.full(len(modis_time), long_missing_value, dtype=np.int64) + for n, t in enumerate(modis_time): + obs_time[n] = round(t) # uncertainty estimates: # From MODIS file (over ocean) and Levy, 2010 (over land) @@ -125,38 +135,17 @@ def _read(self): over_ocean = np.logical_not(land_sea_flag > 0) over_land = np.logical_not(land_sea_flag == 0) UNC = np.where(over_land, unc_land, np.add(0.05, np.multiply(0.15, aod))) - if first: - self.outdata[('latitude', 'MetaData')] = lats - self.outdata[('longitude', 'MetaData')] = lons - self.outdata[('dateTime', 'MetaData')] = obs_time - else: - self.outdata[('latitude', 'MetaData')] = np.concatenate((self.outdata[('latitude', 'MetaData')], lats)) - self.outdata[('longitude', 'MetaData')] = np.concatenate((self.outdata[('longitude', 'MetaData')], lons)) - self.outdata[('dateTime', 'MetaData')] = np.concatenate((self.outdata[('dateTime', 'MetaData')], obs_time)) - - for ncvar, iodavar in obsvars.items(): - data = aod.astype('float32') - err = UNC.astype('float32') - data2d = np.array(data).reshape(len(data), 1) - err2d = np.array(err).reshape(len(err), 1) - qc2d = np.array(QC_flag).reshape(len(QC_flag), 1) - if first: - self.outdata[self.varDict[iodavar]['valKey']] = data2d - self.outdata[self.varDict[iodavar]['errKey']] = err - self.outdata[self.varDict[iodavar]['qcKey']] = QC_flag - - else: - self.outdata[self.varDict[iodavar]['valKey']] = np.concatenate( - (self.outdata[self.varDict[iodavar]['valKey']], data)) - self.outdata[self.varDict[iodavar]['errKey']] = np.concatenate( - (self.outdata[self.varDict[iodavar]['errKey']], err)) - self.outdata[self.varDict[iodavar]['qcKey']] = np.concatenate( - (self.outdata[self.varDict[iodavar]['qcKey']], QC_flag)) - - first = False - self.varAttrs[('dateTime', 'MetaData')]['units'] = '' - DimDict['Location'] = len(self.outdata[('dateTime', 'MetaData')]) - DimDict['Channel'] = np.array([2], dtype=np.intc) + + self.outdata[('latitude', metaDataName)] = np.append(self.outdata[('latitude', metaDataName)], np.array(lats, dtype=np.float32)) + self.outdata[('longitude', metaDataName)] = np.append(self.outdata[('longitude', metaDataName)], np.array(lons, dtype=np.float32)) + self.outdata[('dateTime', metaDataName)] = np.append(self.outdata[('dateTime', metaDataName)], np.array(obs_time, dtype=np.int64)) + + for iodavar in obsvars: + self.outdata[self.varDict[iodavar]['valKey']] = np.append(self.outdata[self.varDict[iodavar]['valKey']], np.array(aod, dtype=np.float32)) + self.outdata[self.varDict[iodavar]['errKey']] = np.append(self.outdata[self.varDict[iodavar]['errKey']], np.array(UNC, dtype=np.float32)) + self.outdata[self.varDict[iodavar]['qcKey']] = np.append(self.outdata[self.varDict[iodavar]['qcKey']], np.array(QC_flag, dtype=np.int32)) + + DimDict['Location'] = len(self.outdata[('dateTime', metaDataName)]) def main(): From c245ce1ad7cb732a8d2c16bc10f92a4127ecca15 Mon Sep 17 00:00:00 2001 From: bhuang95 Date: Fri, 11 Mar 2022 16:55:14 +0000 Subject: [PATCH 5/6] Update testoutput/modis_aod.nc after Greg's change --- test/testoutput/modis_aod.nc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testoutput/modis_aod.nc b/test/testoutput/modis_aod.nc index e56b28215..ecd874a4f 100644 --- a/test/testoutput/modis_aod.nc +++ b/test/testoutput/modis_aod.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:60ba78c59d21f80d8f52d4bdb465d0543d8cfc600541b0a8bb7ee96301475891 -size 34398 +oid sha256:36f2bab199b0648bb749c98c9697287245c0f0608df249bc14050fa8be31b67f +size 18355 From a1d8dc2a808701d9fe9f1120c919a55e5362c0dd Mon Sep 17 00:00:00 2001 From: bhuang95 Date: Fri, 11 Mar 2022 21:10:09 +0000 Subject: [PATCH 6/6] Update modis aod test reference files --- test/testoutput/modis_aod.nc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testoutput/modis_aod.nc b/test/testoutput/modis_aod.nc index ecd874a4f..9f8eb13e1 100644 --- a/test/testoutput/modis_aod.nc +++ b/test/testoutput/modis_aod.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:36f2bab199b0648bb749c98c9697287245c0f0608df249bc14050fa8be31b67f -size 18355 +oid sha256:0622a99d75d1cd0a280e6c86ac3cd1b4d601e2b504c7b4d94d85bff9e936a72d +size 18923