Create Feature/sprint ghcn for 6 land converters during the ioda-converter code spring

src/land/ghcn_snod2ioda.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# (C) Copyright 2021 NOAA/NWS/NCEP/EMC
+# (C) Copyright 2021-2022 NOAA/NWS/NCEP/EMC
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 #
@@ -27,22 +27,21 @@
     ("latitude", "float"),
     ("longitude", "float"),
     ("height", "float"),
-    ("datetime", "string")
+    ("dateTime", "string")
 ]
 
 obsvars = {
     'snow_depth': 'totalSnowDepth',
 }
 
 AttrData = {
-    'converter': os.path.basename(__file__),
 }
 
 DimDict = {
 }
 
 VarDims = {
-    'totalSnowDepth': ['nlocs'],
+    'totalSnowDepth': ['Location'],
 }
 
 
@@ -70,9 +69,8 @@ def _read(self):
         self.varAttrs[iodavar, iconv.OvalName()]['coordinates'] = 'longitude latitude'
         self.varAttrs[iodavar, iconv.OerrName()]['coordinates'] = 'longitude latitude'
         self.varAttrs[iodavar, iconv.OqcName()]['coordinates'] = 'longitude latitude'
-        self.varAttrs[iodavar, iconv.OvalName()]['units'] = 'mm'
-        self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'mm'
-        self.varAttrs[iodavar, iconv.OqcName()]['units'] = 'unitless'
+        self.varAttrs[iodavar, iconv.OvalName()]['units'] = 'm'
+        self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'm'
 
         def assignValue(colrowValue, df400):
             if colrowValue == '' or pd.isnull(colrowValue):
@@ -162,14 +160,14 @@ def assignValue(colrowValue, df400):
         my_date = datetime.strptime(startdate, "%Y%m%d")
         start_datetime = my_date.strftime('%Y-%m-%d')
         base_datetime = start_datetime + 'T18:00:00Z'
-        AttrData['date_time_string'] = base_datetime
 
         for i in range(len(vals)):
             if vals[i] >= 0.0:
-                errs[i] = 40.0
+                errs[i] = 0.04
+                vals[i] = 0.001*vals[i]
             times[i] = base_datetime
         # add metadata variables
-        self.outdata[('datetime', 'MetaData')] = times
+        self.outdata[('dateTime', 'MetaData')] = times
         self.outdata[('stationIdentification', 'MetaData')] = sites
         self.outdata[('latitude', 'MetaData')] = lats
         self.outdata[('longitude', 'MetaData')] = lons
@@ -180,8 +178,7 @@ def assignValue(colrowValue, df400):
         self.outdata[self.varDict[iodavar]['errKey']] = errs
         self.outdata[self.varDict[iodavar]['qcKey']] = qflg
 
-        DimDict['nlocs'] = len(self.outdata[('datetime', 'MetaData')])
-        AttrData['nlocs'] = np.int32(DimDict['nlocs'])
+        DimDict['Location'] = len(self.outdata[('dateTime', 'MetaData')])
 
 
 def main():

src/land/imsfv3_scf2ioda.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# (C) Copyright 2021 NOAA/NWS/NCEP/EMC
+# (C) Copyright 2020-2022 NOAA/NWS/NCEP/EMC
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -28,7 +28,7 @@
     ("latitude", "float"),
     ("longitude", "float"),
     ("height", "float"),
-    ("datetime", "string")
+    ("dateTime", "string")
 ]
 
 obsvars = {
@@ -37,15 +37,14 @@
 }
 
 AttrData = {
-    'converter': os.path.basename(__file__),
 }
 
 DimDict = {
 }
 
 VarDims = {
-    'snowCoverFraction': ['nlocs'],
-    'totalSnowDepth': ['nlocs'],
+    'snowCoverFraction': ['Location'],
+    'totalSnowDepth': ['Location'],
 }
 
 
@@ -71,22 +70,19 @@ def _read(self):
             if iodavar == 'snowCoverFraction':
                 self.varAttrs[iodavar, iconv.OvalName()]['units'] = '1'
                 self.varAttrs[iodavar, iconv.OerrName()]['units'] = '1'
-                self.varAttrs[iodavar, iconv.OqcName()]['units'] = 'unitless'
                 self.varAttrs[iodavar, iconv.OvalName()]['_FillValue'] = -999.
                 self.varAttrs[iodavar, iconv.OerrName()]['_FillValue'] = -999.
                 self.varAttrs[iodavar, iconv.OqcName()]['_FillValue'] = -999
 
             if iodavar == 'totalSnowDepth':
-                self.varAttrs[iodavar, iconv.OvalName()]['units'] = 'mm'
-                self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'mm'
-                self.varAttrs[iodavar, iconv.OqcName()]['units'] = 'unitless'
+                self.varAttrs[iodavar, iconv.OvalName()]['units'] = 'm'
+                self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'm'
                 self.varAttrs[iodavar, iconv.OvalName()]['_FillValue'] = -999.
                 self.varAttrs[iodavar, iconv.OerrName()]['_FillValue'] = -999.
                 self.varAttrs[iodavar, iconv.OqcName()]['_FillValue'] = -999
 
         # read netcdf file
         ncd = nc.Dataset(self.filename)
-        AttrData["sensor"] = "IMS Multisensor"
         lons = ncd.variables['lon'][:]
         lats = ncd.variables['lat'][:]
         oros = ncd.variables['oro'][:]
@@ -103,7 +99,7 @@ def _read(self):
         qdflg = 0*sndv.astype('int32')
         errsc = 0.0*sncv
         errsd = 0.0*sndv
-        errsd[:] = 80.0
+        errsd[:] = 0.08
         ncd.close()
 
         times = np.empty_like(sncv, dtype=object)
@@ -113,13 +109,13 @@ def _read(self):
         my_date = datetime.strptime(str_date, "%Y%m%d")
         start_datetime = my_date.strftime('%Y-%m-%d')
         base_datetime = start_datetime + 'T18:00:00Z'
-        AttrData['date_time_string'] = base_datetime
 
         for i in range(len(lats)):
             times[i] = base_datetime
+            sndv[i] = 0.001*sndv[i]
 
         # add metadata variables
-        self.outdata[('datetime', 'MetaData')] = times
+        self.outdata[('dateTime', 'MetaData')] = times
         self.outdata[('latitude', 'MetaData')] = lats
         self.outdata[('longitude', 'MetaData')] = lons
         self.outdata[('height', 'MetaData')] = oros
@@ -136,8 +132,7 @@ def _read(self):
                     self.outdata[self.varDict[iodavar]['valKey']] = sndv
                     self.outdata[self.varDict[iodavar]['errKey']] = errsd
                     self.outdata[self.varDict[iodavar]['qcKey']] = qdflg
-        DimDict['nlocs'] = len(self.outdata[('datetime', 'MetaData')])
-        AttrData['nlocs'] = np.int32(DimDict['nlocs'])
+        DimDict['Location'] = len(self.outdata[('dateTime', 'MetaData')])
 
 
 def main():

src/land/owp_snow_obs.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# (C) Copyright 2019 UCAR
+# (C) Copyright 2019-2022 UCAR
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 
@@ -32,11 +32,11 @@
     to IODA output files. """)
 
 # obs file name -> ioda file name
-output_var_dict = {'snow_depth_m': 'snow_depth', 'snow_water_equivalent_mm': 'swe'}
+output_var_dict = {'snow_depth_m': 'totalSnowDepth', 'snow_water_equivalent_mm': 'snowWaterEquivalent'}
 # ioda file_name -> ioda file units
-output_var_unit_dict = {'snow_depth': 'm', 'swe': 'mm'}
+output_var_unit_dict = {'totalSnowDepth': 'm', 'snowWaterEquivalent': 'kg m-2'}
 one = 1.00000000000000
-output_conversion_factor = {'snow_depth': one, 'swe': one}
+output_conversion_factor = {'totalSnowDepth': one, 'snowWaterEquivalent': one}
 
 col_types = {
     'StnObjID': np.int32,
@@ -48,25 +48,23 @@
     'rec_elev_m': np.float32,
     'latitude': np.float64,
     'longitude': np.float64,
-    'datetime': str,
+    'dateTime': str,
     'variable_name': str}
 
 location_key_list = [
     ("latitude", "float"),
     ("longitude", "float"),
     ("height", "integer"),
-    ("datetime", "string"), ]
+    ("dateTime", "string"), ]
 
 dim_dict = {}
 
 var_dims = {
-    'snow_depth': ['nlocs'],
-    'swe': ['nlocs'], }
+    'totalSnowDepth': ['Location'],
+    'snowWaterEquivalent': ['Location'], }
 
 attr_data = {
-    'converter': os.path.basename(__file__),
-    'converter_version': 0.3,
-    'nvars': np.int32(len(var_dims)), }
+ }
 
 fill_value = 9.96921e+36
 
@@ -108,7 +106,6 @@ def __init__(
 
     def _read(self):
         # print(f"Reading: {self.file_in}")
-        self.attr_data['obs_file'] = str(self.file_in.split('/')[-1])
         # use pandas to get the data lined up
         obs_df = pd.read_csv(self.file_in, header=0, index_col=False, dtype=col_types)
 
@@ -135,7 +132,7 @@ def _read(self):
             .sort_index()
             .reset_index())
 
-        self.attr_data['ref_date_time'] = (
+        self.attr_data['datetimeReference'] = (
             pd.to_datetime(obs_df.datetime[0])
             .round('D')
             .strftime('%Y-%m-%dT%H:%M:%SZ'))
@@ -173,14 +170,13 @@ def _read(self):
                     wh_not_var_other = np.where(np.isnan(obs_df[f'ObsValue {var_other}']))[0]  # 1-D
                     obs_df = obs_df.drop(wh_not_var_other).reset_index()
 
-        self.data[('datetime', 'MetaData')] = obs_df.datetime.values
+        self.data[('dateTime', 'MetaData')] = obs_df.datetime.values
         self.data[('latitude', 'MetaData')] = obs_df.latitude.values.astype('float32')
         self.data[('longitude', 'MetaData')] = obs_df.longitude.values.astype('float32')
 
         self.data[('height', 'MetaData')] = obs_df.rec_elev_m.values.astype('float32')
-        self.data[('station_id', 'MetaData')] = obs_df.StnID.values
+        self.data[('stationIdentification', 'MetaData')] = obs_df.StnID.values
         self.var_metadata[('height', 'MetaData')]['units'] = 'm'
-        self.var_metadata[('station_id', 'MetaData')]['units'] = 'unitless'
 
         for obs_var, ioda_var in output_var_dict.items():
             # define the ioda variable
@@ -190,9 +186,9 @@ def _read(self):
             # define ioda meta/ancillary
             for name in [iconv.OvalName(), iconv.OerrName(), iconv.OqcName()]:
                 self.var_metadata[ioda_var, name]['coordinates'] = 'longitude latitude'
-                self.var_metadata[ioda_var, name]['units'] = output_var_unit_dict[ioda_var]  # not really for Oqc... but
+                if(iconv.OqcName()!=name):
+                    self.var_metadata[ioda_var, name]['units'] = output_var_unit_dict[ioda_var]  # not really for Oqc... but
             # just kidding for OqcName... a lazy tag along above, fix now (less code to overwrite)
-            self.var_metadata[ioda_var, iconv.OqcName()]['units'] = 'unitless'
             # the data
             conv_fact = output_conversion_factor[ioda_var]
             self.data[self.var_dict[ioda_var]['valKey']] = (
@@ -206,9 +202,7 @@ def _read(self):
             self.data[self.var_dict[ioda_var]['qcKey']] = (
                 mask_nans(obs_df[f'PreQC {obs_var}'].values * conv_fact))
 
-        nlocs = len(self.data[('datetime', 'MetaData')])
-        dim_dict['nlocs'] = nlocs
-        attr_data['nlocs'] = np.int32(nlocs)
+        dim_dict['Location'] = len(self.data[('dateTime', 'MetaData')])
 
     def write(self):
         writer = iconv.IodaWriter(self.file_out, location_key_list, dim_dict)

src/land/smap9km_ssm2ioda.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# (C) Copyright 2022 EMC/NCEP/NWS/NOAA
+# (C) Copyright 2020-2022 EMC/NCEP/NWS/NOAA
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -25,22 +25,21 @@
 locationKeyList = [
     ("latitude", "float"),
     ("longitude", "float"),
-    ("datetime", "string")
+    ("dateTime", "string")
 ]
 
 obsvars = {
     'soil_moisture': 'soilMoistureVolumetric',
 }
 
 AttrData = {
-    'converter': os.path.basename(__file__),
 }
 
 DimDict = {
 }
 
 VarDims = {
-    'soilMoistureVolumetric': ['nlocs'],
+    'soilMoistureVolumetric': ['Location'],
 }
 
 
@@ -65,20 +64,18 @@ def _read(self):
             self.varAttrs[iodavar, iconv.OqcName()]['coordinates'] = 'longitude latitude'
             self.varAttrs[iodavar, iconv.OvalName()]['units'] = 'm3 m-3'
             self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'm3 m-3'
-            self.varAttrs[iodavar, iconv.OqcName()]['units'] = 'unitless'
 
         # open input file name
         ncd = nc.Dataset(self.filename, 'r')
         # set and get global attributes
-        self.satellite = "SMAP"
-        self.sensor = "radar and radiometer"
-        AttrData["satellite"] = self.satellite
-        AttrData["sensor"] = self.sensor
+        self.satellite = 789
+        self.sensor = 432
+        AttrData["platform"] = np.array([self.satellite], dtype=np.int32)
+        AttrData["sensor"] = np.array([self.sensor], dtype=np.int32)
 
         data = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['soil_moisture'][:]
         vals = data[:].ravel()
         _FillValue = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['soil_moisture'].getncattr('_FillValue')
-        self.varAttrs['soilMoistureVolumetric', iconv.OvalName()]['_FillValue'] = _FillValue
         valid_max = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['soil_moisture'].getncattr('valid_max')
         valid_min = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['soil_moisture'].getncattr('valid_min')
 
@@ -115,29 +112,25 @@ def _read(self):
         qflg = qflg.astype('int32')
         sflg = sflg.astype('int32')
         vegop = vegop.astype('int32')
-        AttrData['date_time_string'] = base_datetime
 
         for i in range(len(lons)):
             times[i] = base_datetime
             errs[i] = 0.04
         # add metadata variables
-        self.outdata[('datetime', 'MetaData')] = times
+        self.outdata[('dateTime', 'MetaData')] = times
         self.outdata[('latitude', 'MetaData')] = lats
         self.outdata[('longitude', 'MetaData')] = lons
         self.varAttrs[('latitude', 'MetaData')]['units'] = 'degree_north'
         self.varAttrs[('longitude', 'MetaData')]['units'] = 'degree_east'
-        self.outdata[('surfaceFlag', 'MetaData')] = sflg
-        self.varAttrs[('surfaceFlag', 'MetaData')]['units'] = 'unitless'
+        self.outdata[('earthSurfaceType', 'MetaData')] = sflg
         self.outdata[('vegetationOpacity', 'MetaData')] = vegop
-        self.varAttrs[('vegetationOpacity', 'MetaData')]['units'] = 'unitless'
 
         for iodavar in ['soilMoistureVolumetric']:
             self.outdata[self.varDict[iodavar]['valKey']] = vals
             self.outdata[self.varDict[iodavar]['errKey']] = errs
             self.outdata[self.varDict[iodavar]['qcKey']] = qflg
 
-        DimDict['nlocs'] = len(self.outdata[('datetime', 'MetaData')])
-        AttrData['nlocs'] = np.int32(DimDict['nlocs'])
+        DimDict['Location'] = len(self.outdata[('dateTime', 'MetaData')])
 
 
 def main():