Sprint: Updating two GOES and three marine converters for naming conventions

src/goes/goes_latlon.py

@@ -13,17 +13,17 @@
 #
 # /GROUP/VARIABLE -> ATTRIBUTE
 #
-# /MetaData/elevation_angle
-# /MetaData/scan_angle
-# /MetaData/scan_position
-# /MetaData/sensor_azimuth_angle -> units
-# /MetaData/sensor_view_angle -> units
-# /MetaData/sensor_zenith_angle -> units
+# /MetaData/sensorElevationAngle
+# /MetaData/sensorScanAngle
+# /MetaData/sensorScanPosition
+# /MetaData/sensorAzimuthAngle
+# /MetaData/sensorViewAngle
+# /MetaData/sensorZenithAngle
 # /MetaData/latitude
 # /MetaData/latitude -> lat_nadir
 # /MetaData/longitude
 # /MetaData/longitude -> lon_nadir
-# /nlocs
+# /Location
 #
 from netCDF4 import Dataset
 from numpy import ma
@@ -192,31 +192,31 @@ def create(self):
         sensor_zenith_angle, sensor_azimuth_angle, sensor_view_angle = \
             self._calc_sensor_zenith_azimuth_view_angles(latitude, longitude)
         latlon_dataset = Dataset(self._latlon_file_path, 'w')
-        nlocs = len(latitude)
-        latlon_dataset.createDimension('nlocs', nlocs)
-        latlon_dataset.createVariable('nlocs', 'i4', ('nlocs',))
-        latlon_dataset.variables['nlocs'][:] = np.arange(1, nlocs + 1, 1, dtype='int32')
+        Location = len(latitude)
+        latlon_dataset.createDimension('Location', Location)
+        latlon_dataset.createVariable('Location', 'i8', ('Location',))
+        latlon_dataset.variables['Location'][:] = np.arange(1, Location + 1, 1, dtype='int64')
         nonexistent_indices = len(self._lat_fill_value_index_array[0])
         latlon_dataset.createDimension('nonexistent_indices', nonexistent_indices)
         latlon_dataset.createVariable('nonexistent_indices', 'i4', ('nonexistent_indices',))
         latlon_dataset.variables['nonexistent_indices'][:] = np.array(self._lat_fill_value_index_array[0])
         latlon_dataset.createGroup('MetaData')
-        latlon_dataset.createVariable('/MetaData/latitude', 'f4', 'nlocs', fill_value=-999)
-        latlon_dataset.createVariable('/MetaData/longitude', 'f4', 'nlocs', fill_value=-999)
-        latlon_dataset.createVariable('/MetaData/scan_angle', 'f4', 'nlocs', fill_value=-999)
-        latlon_dataset.createVariable('/MetaData/scan_position', 'i4', 'nlocs', fill_value=-999)
-        latlon_dataset.createVariable('/MetaData/elevation_angle', 'f4', 'nlocs', fill_value=-999)
-        latlon_dataset.createVariable('/MetaData/sensor_zenith_angle', 'f4', 'nlocs', fill_value=-999)
-        latlon_dataset.createVariable('/MetaData/sensor_azimuth_angle', 'f4', 'nlocs', fill_value=-999)
-        latlon_dataset.createVariable('/MetaData/sensor_view_angle', 'f4', 'nlocs', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/latitude', 'f4', 'Location', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/longitude', 'f4', 'Location', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/sensorScanAngle', 'f4', 'Location', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/sensorScanPosition', 'i4', 'Location', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/sensorElevationAngle', 'f4', 'Location', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/sensorZenithAngle', 'f4', 'Location', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/sensorAzimuthAngle', 'f4', 'Location', fill_value=-999)
+        latlon_dataset.createVariable('/MetaData/sensorViewAngle', 'f4', 'Location', fill_value=-999)
         latlon_dataset['/MetaData/latitude'][:] = latitude
         latlon_dataset['/MetaData/longitude'][:] = longitude
-        latlon_dataset['/MetaData/scan_angle'][:] = scan_angle
-        latlon_dataset['/MetaData/scan_position'][:] = scan_position
-        latlon_dataset['/MetaData/elevation_angle'][:] = elevation_angle
-        latlon_dataset['/MetaData/sensor_zenith_angle'][:] = sensor_zenith_angle
-        latlon_dataset['/MetaData/sensor_azimuth_angle'][:] = sensor_azimuth_angle
-        latlon_dataset['/MetaData/sensor_view_angle'][:] = sensor_view_angle
+        latlon_dataset['/MetaData/sensorScanAngle'][:] = scan_angle
+        latlon_dataset['/MetaData/sensorScanPosition'][:] = scan_position
+        latlon_dataset['/MetaData/sensorElevationAngle'][:] = elevation_angle
+        latlon_dataset['/MetaData/sensorZenithAngle'][:] = sensor_zenith_angle
+        latlon_dataset['/MetaData/sensorAzimuthAngle'][:] = sensor_azimuth_angle
+        latlon_dataset['/MetaData/sensorViewAngle'][:] = sensor_view_angle
         lat_nadir, lon_nadir = self._get_nadir_attributes()
         latlon_dataset['/MetaData/latitude'].setncattr('lat_nadir', lat_nadir)
         latlon_dataset['/MetaData/longitude'].setncattr('lon_nadir', lon_nadir)

src/marine/smos_sss2ioda.py

@@ -28,7 +28,7 @@
 locationKeyList = [
     ("latitude", "float"),
     ("longitude", "float"),
-    ("dateTime", "string")
+    ("dateTime", "long")
 ]
 
 GlobalAttrs = {}
@@ -132,8 +132,8 @@ def main():
     writer = iconv.IodaWriter(args.output, locationKeyList, DimDict)
 
     VarAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict))
-    VarAttrs[('seaSurfaceSalinity', 'ObsValue')]['units'] = '1'
-    VarAttrs[('seaSurfaceSalinity', 'ObsError')]['units'] = '1'
+    VarAttrs[('seaSurfaceSalinity', 'ObsValue')]['units'] = 'PSU'
+    VarAttrs[('seaSurfaceSalinity', 'ObsError')]['units'] = 'PSU'
     VarAttrs[('seaSurfaceSalinity', 'ObsValue')]['_FillValue'] = 999
     VarAttrs[('seaSurfaceSalinity', 'ObsError')]['_FillValue'] = 999
     VarAttrs[('seaSurfaceSalinity', 'PreQC')]['_FillValue'] = 999

src/marine/viirs_modis_l2_oc2ioda.py

@@ -39,7 +39,7 @@
 locationKeyList = [
     ("latitude", "float"),
     ("longitude", "float"),
-    ("dateTime", "string"),
+    ("dateTime", "long"),
 ]
 
 
@@ -92,6 +92,7 @@ def read_input(input_args):
     time = (np.repeat(sla.variables['msec'][:].ravel(),
             pixels_per_line).ravel() - sla.variables['msec'][0])/1000.0
     data_in['time'] = time[mask]
+    time_units = basetime.strftime("%Y-%m-%dT%H:%M:%SZ")
 
     # load in all the other data and apply the missing value mask
     input_vars = ('poc', 'chlor_a')
@@ -116,8 +117,7 @@ def read_input(input_args):
     # create a string version of the date for each observation
     dates = []
     for i in range(len(lons)):
-        obs_date = basetime + timedelta(seconds=float(data_in['time'][i]))
-        dates.append(obs_date.strftime("%Y-%m-%dT%H:%M:%SZ"))
+        dates.append(np.int64(data_in['time'][i]))
 
     # allocate space for output depending on which variables are to be saved
     obs_dim = (len(lons))
@@ -138,7 +138,7 @@ def read_input(input_args):
             np.zeros(obs_dim)
 
     # Add the metadata
-    obs_data[('dateTime', 'MetaData')] = np.empty(len(dates), dtype=object)
+    obs_data[('dateTime', 'MetaData')] = np.empty(len(dates), dtype=np.int64)
     obs_data[('dateTime', 'MetaData')][:] = dates
     obs_data[('latitude', 'MetaData')] = lats
     obs_data[('longitude', 'MetaData')] = lons
@@ -158,7 +158,7 @@ def read_input(input_args):
         obs_data[output_var_names[1], global_config['opqc_name']] = \
             data_in['l2_flags']
 
-    return (obs_data, GlobalAttrs)
+    return (obs_data, GlobalAttrs, time_units)
 
 
 def main():
@@ -236,7 +236,7 @@ def main():
     obs = pool.map(read_input, pool_inputs)
 
     # concatenate the data from the files
-    obs_data, GlobalAttrs = obs[0]
+    obs_data, GlobalAttrs, time_units = obs[0]
     for i in range(1, len(obs)):
         obs_data.update(obs[i][0])
     # Get the Location
@@ -249,6 +249,10 @@ def main():
     GlobalAttrs['converter'] = os.path.basename(__file__)
     DimDict['Location'] = Location
 
+    VarAttrs[('dateTime', 'MetaData')]['units'] = time_units
+    VarAttrs[('latitude', 'MetaData')]['units'] = 'degrees_north'
+    VarAttrs[('longitude', 'MetaData')]['units'] = 'degrees_east'
+
     # determine which variables we are going to output
     if args.poc:
         VarAttrs[output_var_names[0], global_config['oval_name']]['units'] = \

src/marine/viirs_modis_l3_oc2ioda.py

@@ -38,11 +38,15 @@
 locationKeyList = [
     ("latitude", "float"),
     ("longitude", "float"),
-    ("dateTime", "string")
+    ("dateTime", "long")
 ]
 
 GlobalAttrs = {}
 
+# Prepare dateTime info
+iso8601_string = '1970-01-01T00:00:00Z'
+epoch = datetime.fromisoformat(iso8601_string[:-1])
+
 
 class OCL3(object):
 
@@ -71,10 +75,14 @@ def _read(self):
         GlobalAttrs['description'] = str(ncd.getncattr('processing_level')+' processing')
 
         timevar = ncd.getncattr('time_coverage_start')
-        obstime = timevar[:19]+'Z'
+        this_time = datetime.fromisoformat(timevar[:19])
+        obstime = np.int64(round((this_time - epoch).total_seconds()))
 
         ncd.close()
 
+        # Convert obstime from string to seconds since blah blah
+
+
         valKey = vName['chlor_a'], iconv.OvalName()
         errKey = vName['chlor_a'], iconv.OerrName()
         qcKey = vName['chlor_a'], iconv.OqcName()
@@ -85,6 +93,10 @@ def _read(self):
         self.VarAttrs[vName['chlor_a'], iconv.OvalName()]['units'] = 'mg m^-3'
         self.VarAttrs[vName['chlor_a'], iconv.OerrName()]['units'] = 'mg m^-3'
 
+        self.VarAttrs[('dateTime', 'MetaData')]['units'] = 'seconds since ' + iso8601_string
+        self.VarAttrs[('latitude', 'MetaData')]['units'] = 'degrees_north'
+        self.VarAttrs[('longitude', 'MetaData')]['units'] = 'degrees_east'
+
         # apply thinning mask
         if self.thin > 0.0:
             mask_thin = np.random.uniform(size=len(lons)) > self.thin