Merge pull request #248 from NREL/gb/qa_fix

Gb/qa fix

Author: grantbuster

sup3r/qa/qa.py

@@ -49,6 +49,7 @@ def __init__(
         t_enhance,
         temporal_coarsening_method,
         features=None,
+        source_features=None,
         output_names=None,
         input_handler_name=None,
         input_handler_kwargs=None,
@@ -85,6 +86,17 @@ def __init__(
             Explicit list of features to validate. Can be a single feature str,
             list of string feature names, or None for all features found in the
             out_file_path.
+        source_features : str | list | None
+            Optional feature names to retrieve from the source dataset if the
+            source feature names are not the same as the sup3r output feature
+            names. These will be used to derive the features to be validated.
+            e.g. If model output is temperature_2m, and these were derived from
+            temperature_min_2m (and max), then source features should be
+            temperature_min_2m and temperature_max_2m while the model output
+            temperature_2m is aggregated using min/max in the
+            temporal_coarsening_method. Another example is features="ghi",
+            source_features="rsds", where this is a simple alternative name
+            lookup.
         output_names : str | list
             Optional output file dataset names corresponding to the features
             list input
@@ -125,6 +137,11 @@ class for argument details.
         self._features = (
             features if isinstance(features, (list, tuple)) else [features]
         )
+        self._source_features = (
+            source_features
+            if isinstance(source_features, (list, tuple))
+            else [source_features]
+        )
         self._out_names = (
             output_names
             if isinstance(output_names, (list, tuple))
@@ -198,6 +215,14 @@ def output_names(self):
             return self.features
         return self._out_names
 
+    @property
+    def source_features(self):
+        """Get a list of source dataset names corresponding to the input source
+        data """
+        if self._source_features is None or self._source_features == [None]:
+            return self.features
+        return self._source_features
+
     @property
     def output_type(self):
         """Get output data type
@@ -451,16 +476,17 @@ def run(self):
         """
 
         errors = {}
-        ziter = zip(self.features, self.output_names)
-        for idf, (feature, dset_out) in enumerate(ziter):
+        ziter = zip(self.features, self.source_features, self.output_names)
+        for idf, (feature, source_feature, dset_out) in enumerate(ziter):
             logger.info(
-                'Running QA on dataset {} of {} for "{}"'.format(
-                    idf + 1, len(self.features), feature
+                'Running QA on dataset {} of {} for feature "{}" '
+                'with source feature name "{}"'.format(
+                    idf + 1, len(self.features), feature, source_feature,
                 )
             )
             data_syn = self.get_dset_out(feature)
             data_syn = self.coarsen_data(idf, feature, data_syn)
-            data_true = self.input_handler[feature][...]
+            data_true = self.input_handler[source_feature][...]
 
             if data_syn.shape != data_true.shape:
                 msg = (

sup3r/solar/solar.py

@@ -138,11 +138,11 @@ def preflight(self):
         assert 'surface_pressure' in self.nsrdb.dsets
         assert isinstance(self.nsrdb_tslice, slice)
 
-        ti_gan = self.gan_data.time_index
-        delta = pd.Series(ti_gan[1:] - ti_gan[:-1]).mean().total_seconds()
+        delta = pd.Series(self.time_index[1:] - self.time_index[:-1])
+        delta = delta.mean().total_seconds()
         msg = (
             'Its assumed that the sup3r GAN output solar data will be '
-            'hourly but received time index: {}'.format(ti_gan)
+            'hourly but received time index: {}'.format(self.time_index)
         )
         assert delta == 3600, msg
 
@@ -305,9 +305,8 @@ def ghi(self):
         """
         if self._ghi is None:
             logger.debug('Calculating GHI.')
-            self._ghi = (
-                self.get_nsrdb_data('clearsky_ghi') * self.clearsky_ratio
-            )
+            nsrdb_cs_ghi = self.get_nsrdb_data('clearsky_ghi')
+            self._ghi = nsrdb_cs_ghi * self.clearsky_ratio
             self._ghi[:, self.out_of_bounds] = 0
         return self._ghi
 

sup3r/utilities/cli.py

@@ -73,10 +73,12 @@ def from_config(cls, module_name, module_class, ctx, config_file, verbose,
         cmd = module_class.get_node_cmd(config)
 
         if hardware_option.lower() in AVAILABLE_HARDWARE_OPTIONS:
-            cls.kickoff_slurm_job(module_name, ctx, pipeline_step, cmd,
+            cls.kickoff_slurm_job(module_name, ctx, cmd,
+                                  pipeline_step=pipeline_step,
                                   **exec_kwargs)
         else:
-            cls.kickoff_local_job(module_name, ctx, cmd, pipeline_step)
+            cls.kickoff_local_job(module_name, ctx, cmd,
+                                  pipeline_step=pipeline_step)
 
     @classmethod
     def from_config_preflight(cls, module_name, ctx, config_file, verbose):