bc smoothing option added for fwp runtime

Author: bnb32

sup3r/bias/bias_calc.py

@@ -30,8 +30,8 @@ class DataRetrievalBase:
     """
 
     def __init__(self, base_fps, bias_fps, base_dset, bias_feature,
-                 target, shape,
-                 base_handler='Resource', bias_handler='DataHandlerNCforCC',
+                 target=None, shape=None, base_handler='Resource',
+                 bias_handler='DataHandlerNCforCC',
                  bias_handler_kwargs=None, decimals=None):
         """
         Parameters
@@ -53,8 +53,10 @@ def __init__(self, base_fps, bias_fps, base_dset, bias_feature,
             be a single feature name corresponding to base_dset
         target : tuple
             (lat, lon) lower left corner of raster to retrieve from bias_fps.
+            If None then the lower left corner of the full domain will be used.
         shape : tuple
-            (rows, cols) grid size to retrieve from bias_fps.
+            (rows, cols) grid size to retrieve from bias_fps. If None then the
+            full domain shape will be used.
         base_handler : str
             Name of rex resource handler class to be retrieved from the rex
             library.
@@ -553,8 +555,8 @@ def _run_single(cls, bias_data, base_fps, bias_feature, base_dset,
                                         bias_feature, base_dset)
         return out
 
-    def fill_smooth_extend(self, out, fill_extend=True, smooth_extend=0,
-                           smooth_interior=0):
+    def fill_and_smooth(self, out, fill_extend=True, smooth_extend=0,
+                        smooth_interior=0):
         """Fill data extending beyond the base meta data extent by doing a
         nearest neighbor gap fill. Smooth interior and extended region with
         given smoothing values.
@@ -580,7 +582,7 @@ def fill_smooth_extend(self, out, fill_extend=True, smooth_extend=0,
             deviation for the gaussian_filter kernel
         smooth_interior : float
             Value to use to smooth the scalar/adder data inside of the spatial
-            domain set by the threshold input. This can reduce the affect of
+            domain set by the threshold input. This can reduce the effect of
             extreme values within aggregations over large number of pixels.
             This value is the standard deviation for the gaussian_filter
             kernel.
@@ -599,7 +601,7 @@ def fill_smooth_extend(self, out, fill_extend=True, smooth_extend=0,
 
                 arr_smooth = arr[..., idt]
 
-                needs_fill = (fill_extend or smooth_extend > 0
+                needs_fill = (np.isnan(arr_smooth).any() and fill_extend
                               or smooth_interior > 0)
 
                 if needs_fill:
@@ -770,8 +772,8 @@ def run(self, knn, threshold=0.6, fp_out=None, max_workers=None,
 
         logger.info('Finished calculating bias correction factors.')
 
-        out = self.fill_smooth_extend(out, fill_extend, smooth_extend,
-                                      smooth_interior)
+        out = self.fill_and_smooth(out, fill_extend, smooth_extend,
+                                   smooth_interior)
 
         self.write_outputs(fp_out, out)
 

sup3r/bias/bias_transforms.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 """Bias correction transformation functions."""
-import numpy as np
 import logging
 from warnings import warn
-from rex import Resource
 
+import numpy as np
+from rex import Resource
+from scipy.ndimage.filters import gaussian_filter
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +38,7 @@ def global_linear_bc(input, scalar, adder, out_range=None):
 
 
 def local_linear_bc(input, feature_name, bias_fp, lr_padded_slice,
-                    out_range=None):
+                    out_range=None, smoothing=0):
     """Bias correct data using a simple annual (or multi-year) *scalar +adder
     method on a site-by-site basis.
 
@@ -64,6 +65,11 @@ def local_linear_bc(input, feature_name, bias_fp, lr_padded_slice,
         source shape will be used.
     out_range : None | tuple
         Option to set floor/ceiling values on the output data.
+    smoothing : float
+        Value to use to smooth the scalar/adder data. This can reduce the
+        effect of extreme values within aggregations over large number of
+        pixels.  This value is the standard deviation for the gaussian_filter
+        kernel.
 
     Returns
     -------
@@ -99,6 +105,15 @@ def local_linear_bc(input, feature_name, bias_fp, lr_padded_slice,
     scalar = np.repeat(scalar, input.shape[-1], axis=-1)
     adder = np.repeat(adder, input.shape[-1], axis=-1)
 
+    if smoothing > 0:
+        for idt in range(scalar.shape[-1]):
+            scalar[..., idt] = gaussian_filter(scalar[..., idt],
+                                               smoothing,
+                                               mode='nearest')
+            adder[..., idt] = gaussian_filter(adder[..., idt],
+                                              smoothing,
+                                              mode='nearest')
+
     out = input * scalar + adder
     if out_range is not None:
         out = np.maximum(out, np.min(out_range))
@@ -108,7 +123,8 @@ def local_linear_bc(input, feature_name, bias_fp, lr_padded_slice,
 
 
 def monthly_local_linear_bc(input, feature_name, bias_fp, lr_padded_slice,
-                            time_index, temporal_avg=True, out_range=None):
+                            time_index, temporal_avg=True, out_range=None,
+                            smoothing=0):
     """Bias correct data using a simple monthly *scalar +adder method on a
     site-by-site basis.
 
@@ -145,6 +161,11 @@ def monthly_local_linear_bc(input, feature_name, bias_fp, lr_padded_slice,
         this to False.
     out_range : None | tuple
         Option to set floor/ceiling values on the output data.
+    smoothing : float
+        Value to use to smooth the scalar/adder data. This can reduce the
+        effect of extreme values within aggregations over large number of
+        pixels.  This value is the standard deviation for the gaussian_filter
+        kernel.
 
     Returns
     -------
@@ -189,6 +210,15 @@ def monthly_local_linear_bc(input, feature_name, bias_fp, lr_padded_slice,
         logger.warning(msg)
         warn(msg)
 
+    if smoothing > 0:
+        for idt in range(scalar.shape[-1]):
+            scalar[..., idt] = gaussian_filter(scalar[..., idt],
+                                               smoothing,
+                                               mode='nearest')
+            adder[..., idt] = gaussian_filter(adder[..., idt],
+                                              smoothing,
+                                              mode='nearest')
+
     out = input * scalar + adder
     if out_range is not None:
         out = np.maximum(out, np.min(out_range))

sup3r/solar/solar.py

@@ -7,19 +7,19 @@
 """
 import glob
 import json
+import logging
 import os
+
 import numpy as np
-import logging
 import pandas as pd
-from scipy.spatial import KDTree
 from farms.disc import disc
 from farms.utilities import calc_dhi, dark_night
-from rex import Resource, MultiTimeResource
+from rex import MultiTimeResource, Resource
 from rex.utilities.fun_utils import get_fun_call_str
+from scipy.spatial import KDTree
 
+from sup3r.postprocessing.file_handling import H5_ATTRS, RexOutputs
 from sup3r.utilities import ModuleName
-from sup3r.postprocessing.file_handling import RexOutputs, H5_ATTRS
-
 
 logger = logging.getLogger(__name__)
 
@@ -545,7 +545,6 @@ def write(self, fp_out, features=('ghi', 'dni', 'dhi')):
                                attrs=attrs,
                                chunks=attrs['chunks'])
                 logger.info(f'Added "{feature}" to output file.')
-
             run_attrs = self.gan_data.h5[self._sup3r_fps[0]].global_attrs
             run_attrs['nsrdb_source'] = self._nsrdb_fp
             fh.run_attrs = run_attrs

sup3r/utilities/utilities.py

@@ -1305,7 +1305,7 @@ def get_input_handler_class(file_paths, input_handler_name):
         A list of files to extract raster data from. Each file must have
         the same number of timesteps. Can also pass a string with a
         unix-style file path which will be passed through glob.glob
-    input_handler : str
+    input_handler_name : str
         data handler class to use for input data. Provide a string name to
         match a class in data_handling.py. If None the correct handler will
         be guessed based on file type and time series properties.