Add support for HyperBB truncated serial data output

+ Add option to select HyperBB firmware v1 or v2
    + v1: full output (older firmware)
    + v2: truncated output (newer firmware)
+ Adjusted HyperBB data parser
+ Add support for any type of content in "combobox_" from Setup modal
+ Fix bug in Widget Select Active Timeseries Variables preventing user to adequately select variables after setting up the instrument from the setup button (it would work if the setup was never saved during the current the session). Affected HyperBB, LISST, and Satlantic (e.g. HyperNav) instruments.

Author: doizuc

inlinino/__init__.py

@@ -8,7 +8,7 @@
 import numpy as np
 
 
-__version__ = '2.9.14.gamma'
+__version__ = '2.9.14.delta'
 
 # Setup Logger
 logging.basicConfig(level=logging.DEBUG)

inlinino/gui.py

@@ -469,6 +469,8 @@ def on_new_ts_data(self, data, timestamp, reset=False):
             self.reset_ts(data, reset)
         n = len(data)
         # Update buffers
+        if n == 0: # Nothing to update (won't trigger reset)
+            return
         self._buffer_timestamp.extend(timestamp)
         for i in range(n):
             self._buffer_data[i].extend(data[i])
@@ -889,8 +891,18 @@ def act_save(self):
             elif field_prefix == 'combobox':
                 if getattr(self, f).currentText() == 'on':
                     self.cfg[field_name] = True
-                else:
+                elif getattr(self, f).currentText() == 'off':
                     self.cfg[field_name] = False
+                else:
+                    value = getattr(self, f).currentText()
+                    try:
+                        value = int(value)
+                    except ValueError:
+                        try:
+                            value = float(value)
+                        except ValueError:
+                            pass
+                    self.cfg[field_name] = value
             elif field_prefix == 'cb':
                 self.cfg[field_name] = getattr(self, f).isChecked()
         if self.cfg['module'] == 'dataq':
@@ -1140,19 +1152,23 @@ def __init__(self, uuid, parent=None):
                 else:
                     getattr(self, 'te_' + k).setPlainText(v)
             elif hasattr(self, 'combobox_' + k):
-                if v:
-                    getattr(self, 'combobox_' + k).setCurrentIndex(0)
+                cb = getattr(self, 'combobox_' + k)
+                if isinstance(v, bool):
+                    cb.setCurrentIndex(0 if v else 1)
                 else:
-                    getattr(self, 'combobox_' + k).setCurrentIndex(1)
+                    try:
+                        cb.setCurrentIndex([cb.itemText(i) for i in range(cb.count())].index(str(v)))
+                    except ValueError:
+                        logger.warning(f'Value {v} not available in GUI for parameter {k}. GUI set to GUI default.')
             elif hasattr(self, 'dsb_' + k):
                 # double spin box
-                getattr(self, 'dsb_' + k).setValue(self.cfg[k])
+                getattr(self, 'dsb_' + k).setValue(v)
             elif hasattr(self, 'sb_' + k):
                 # spin box
-                getattr(self, 'sb_' + k).setValue(self.cfg[k])
+                getattr(self, 'sb_' + k).setValue(v)
             elif hasattr(self, 'cb_' + k):
                 # check boxes
-                getattr(self, 'cb_' + k).setChecked(self.cfg[k])
+                getattr(self, 'cb_' + k).setChecked(v)
         # Populate special fields specific to each module
         if self.cfg['module'] == 'dataq':
             for c in self.cfg['channels_enabled']:

inlinino/instruments/hyperbb.py

@@ -21,6 +21,7 @@ def __init__(self, uuid, cfg, signal, *args, **kwargs):
         # Instrument Specific Attributes
         self._parser: Optional[HyperBBParser] = None
         self.signal_reconstructed = None
+        self.invalid_packet_alarm_triggered = False
         # Default serial communication parameters
         self.default_serial_baudrate = 9600
         self.default_serial_timeout = 1
@@ -49,7 +50,9 @@ def setup(self, cfg):
             raise ValueError('Missing calibration plaque file (*.mat)')
         if 'temperature_file' not in cfg.keys():
             raise ValueError('Missing calibration temperature file (*.mat)')
-        self._parser = HyperBBParser(cfg['plaque_file'], cfg['temperature_file'])
+        if 'firmware_version' not in cfg.keys():
+            cfg['firmware_version'] = 1
+        self._parser = HyperBBParser(cfg['plaque_file'], cfg['temperature_file'], cfg['firmware_version'])
         self.signal_reconstructed = np.empty(len(self._parser.wavelength)) * np.nan
         # Overload cfg with received data
         prod_var_names = ['beta_u', 'bb']
@@ -65,13 +68,25 @@ def setup(self, cfg):
         self.spectrum_plot_x_values = [self._parser.wavelength]
         # Update Active Timeseries Variables
         self.widget_active_timeseries_variables_names = ['beta(%d)' % x for x in self._parser.wavelength]
+        self.widget_active_timeseries_variables_selected = []
         self.active_timeseries_wavelength = np.zeros(len(self._parser.wavelength), dtype=bool)
         for wl in np.arange(450, 700, 50):
             channel_name = 'beta(%d)' % self._parser.wavelength[np.argmin(np.abs(self._parser.wavelength - wl))]
             self.update_active_timeseries_variables(channel_name, True)
+        # Reset Alarm
+        self.invalid_packet_alarm_triggered = False
 
     def parse(self, packet):
-        return self._parser.parse(packet)
+        if len(packet) == 0:  # Empty lines on firmware v2 at end of wavelength scan
+            return []
+        data = self._parser.parse(packet)
+        if len(data) == 0:
+            self.signal.packet_corrupted.emit()
+            if self.invalid_packet_alarm_triggered is False:
+                self.invalid_packet_alarm_triggered = True
+                self.logger.warning('Unable to parse frame. Check firmware version.')
+                self.signal.alarm_custom.emit('Unable to parse frame.', 'Check HyperBB firmware version in "Setup".')
+        return data
 
     def handle_data(self, raw, timestamp):
         beta_u, bb, wl, gain, net_ref_zero_flag = self._parser.calibrate(np.array([raw], dtype=float))
@@ -122,27 +137,44 @@ def update_active_timeseries_variables(self, name, state):
             ['beta(%d)' % wl for wl in self._parser.wavelength[self.active_timeseries_wavelength]]
 
 
-class MetaHyperBBParser(type):
-    def __init__(cls, name, bases, dct):
-        cls.FRAME_VARIABLES = ['ScanIdx', 'DataIdx', 'Date', 'Time', 'StepPos', 'wl', 'LedPwr', 'PmtGain', 'NetSig1',
-                               'SigOn1', 'SigOn1Std', 'RefOn', 'RefOnStd', 'SigOff1', 'SigOff1Std', 'RefOff',
-                               'RefOffStd', 'SigOn2', 'SigOn2Std', 'SigOn3', 'SigOn3Std', 'SigOff2', 'SigOff2Std',
-                               'SigOff3', 'SigOff3Std', 'LedTemp', 'WaterTemp', 'Depth', 'Debug1', 'zDistance']
-        cls.FRAME_TYPES = [int, int, str, str, int, int, int, int, int,
-                           float, float, float, float, float, float, float,
-                           float, float, float, float, float, float, float,
-                           float, float, float, float, float, int, int]
-        # FRAME_PRECISIONS = ['%d', '%d', '%s', '%s', '%d', '%d', '%d', '%d', '%d',
-        #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
-        #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
-        #                    '%.1f', '%.1f', '%.2f', '%.2f', '%.2f', '%d', '%d']
-        cls.FRAME_PRECISIONS = ['%s'] * len(cls.FRAME_VARIABLES)
-        for x in cls.FRAME_VARIABLES:
-            setattr(cls, f'idx_{x}', cls.FRAME_VARIABLES.index(x))
-
+class HyperBBParser():
+    def __init__(self, plaque_cal_file, temperature_cal_file, firmware_version=1):
+        # Frame Parser
+        self.firmware_version = firmware_version
+        if firmware_version == 1:
+            self.FRAME_VARIABLES = ['ScanIdx', 'DataIdx', 'Date', 'Time', 'StepPos', 'wl', 'LedPwr', 'PmtGain', 'NetSig1',
+                                   'SigOn1', 'SigOn1Std', 'RefOn', 'RefOnStd', 'SigOff1', 'SigOff1Std', 'RefOff',
+                                   'RefOffStd', 'SigOn2', 'SigOn2Std', 'SigOn3', 'SigOn3Std', 'SigOff2', 'SigOff2Std',
+                                   'SigOff3', 'SigOff3Std', 'LedTemp', 'WaterTemp', 'Depth', 'Debug1', 'zDistance']
+            self.FRAME_TYPES = [int, int, str, str, int, int, int, int, int,
+                               float, float, float, float, float, float, float,
+                               float, float, float, float, float, float, float,
+                               float, float, float, float, float, int, int]
+            # FRAME_PRECISIONS = ['%d', '%d', '%s', '%s', '%d', '%d', '%d', '%d', '%d',
+            #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
+            #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
+            #                    '%.1f', '%.1f', '%.2f', '%.2f', '%.2f', '%d', '%d']
+            self.FRAME_PRECISIONS = ['%s'] * len(self.FRAME_VARIABLES)
+            for x in self.FRAME_VARIABLES:
+                setattr(self, f'idx_{x}', self.FRAME_VARIABLES.index(x))
+        elif firmware_version == 2:
+            self.FRAME_VARIABLES = ['ScanIdx', 'Date', 'Time', 'wl', 'PmtGain',
+                                   'NetRef', 'NetSig1', 'NetSig2', 'NetSig3',
+                                   'LedTemp', 'WaterTemp', 'Depth', 'SupplyVolt', 'ChSaturated']
+            self.FRAME_TYPES = [int, str, str, int, int,
+                                float, float, float, float, float,
+                                float, float, float, float, float]
+            # FRAME_PRECISIONS = ['%d', '%d', '%s', '%s', '%d', '%d', '%d', '%d', '%d',
+            #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
+            #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
+            #                    '%.1f', '%.1f', '%.2f', '%.2f', '%.2f', '%d', '%d']
+            self.FRAME_PRECISIONS = ['%s'] * len(self.FRAME_VARIABLES)
+            for x in self.FRAME_VARIABLES:
+                setattr(self, f'idx_{x}', self.FRAME_VARIABLES.index(x))
+        else:
+            raise ValueError('Firmware version not supported.')
 
-class HyperBBParser(metaclass=MetaHyperBBParser):
-    def __init__(self, plaque_cal_file, temperature_cal_file):
+        # Instrument Specific Attributes
         self._theta = float('nan')
         self.Xp = float('nan')
 
@@ -229,22 +261,29 @@ def calibrate(self, raw):
                     raw = np.delete(raw, sel, axis=0)
         # Shortcuts
         wl = raw[:, self.idx_wl]
-        # Remove saturated reading
-        raw[raw[:, self.idx_SigOn1] > self.saturation_level, self.idx_SigOn1] = np.nan
-        raw[raw[:, self.idx_SigOn2] > self.saturation_level, self.idx_SigOn2] = np.nan
-        raw[raw[:, self.idx_SigOn3] > self.saturation_level, self.idx_SigOn3] = np.nan
-        raw[raw[:, self.idx_SigOff1] > self.saturation_level, self.idx_SigOff1] = np.nan
-        raw[raw[:, self.idx_SigOff2] > self.saturation_level, self.idx_SigOff2] = np.nan
-        raw[raw[:, self.idx_SigOff3] > self.saturation_level, self.idx_SigOff3] = np.nan
-        # Calculate net signal for ref, low gain (2), high gain (3)
-        net_ref = raw[:, self.idx_RefOn] - raw[:, self.idx_RefOff]
-        net_sig2 = raw[:, self.idx_SigOn2] - raw[:, self.idx_SigOff2]
-        net_sig3 = raw[:, self.idx_SigOn3] - raw[:, self.idx_SigOff3]
-        net_ref_zero_flag = np.any(net_ref == 0)
-        net_ref[net_ref == 0] = np.nan
-        scat1 = raw[:, self.idx_NetSig1] / net_ref
-        scat2 = net_sig2 / net_ref
-        scat3 = net_sig3 / net_ref
+        if self.firmware_version == 1:
+            # Remove saturated reading
+            raw[raw[:, self.idx_SigOn1] > self.saturation_level, self.idx_SigOn1] = np.nan
+            raw[raw[:, self.idx_SigOn2] > self.saturation_level, self.idx_SigOn2] = np.nan
+            raw[raw[:, self.idx_SigOn3] > self.saturation_level, self.idx_SigOn3] = np.nan
+            raw[raw[:, self.idx_SigOff1] > self.saturation_level, self.idx_SigOff1] = np.nan
+            raw[raw[:, self.idx_SigOff2] > self.saturation_level, self.idx_SigOff2] = np.nan
+            raw[raw[:, self.idx_SigOff3] > self.saturation_level, self.idx_SigOff3] = np.nan
+            # Calculate net signal for ref, low gain (2), high gain (3)
+            net_ref = raw[:, self.idx_RefOn] - raw[:, self.idx_RefOff]
+            net_sig2 = raw[:, self.idx_SigOn2] - raw[:, self.idx_SigOff2]
+            net_sig3 = raw[:, self.idx_SigOn3] - raw[:, self.idx_SigOff3]
+            net_ref_zero_flag = np.any(net_ref == 0)
+            net_ref[net_ref == 0] = np.nan
+            scat1 = raw[:, self.idx_NetSig1] / net_ref
+            scat2 = net_sig2 / net_ref
+            scat3 = net_sig3 / net_ref
+        else:  # Assume firmware 2
+            net_ref_zero_flag = np.any(raw[:, self.idx_NetRef] == 0)
+            # TODO Check if need to flag saturated (ChSaturated)
+            scat1 = raw[:, self.idx_NetSig1] / raw[:, self.idx_NetRef]
+            scat2 = raw[:, self.idx_NetSig2] / raw[:, self.idx_NetRef]
+            scat3 = raw[:, self.idx_NetSig3] / raw[:, self.idx_NetRef]
         # Subtract dark offset
         scat1_dark_removed = scat1 - self.f_dark_cal_scat_1(raw[:, self.idx_PmtGain], wl)
         scat2_dark_removed = scat2 - self.f_dark_cal_scat_2(raw[:, self.idx_PmtGain], wl)
@@ -264,9 +303,15 @@ def calibrate(self, raw):
         scatx_corrected[np.isnan(scatx_corrected)] = scat2_t_corrected[np.isnan(scatx_corrected)] # otherwise low gain
         scatx_corrected[np.isnan(scatx_corrected)] = scat1_t_corrected[np.isnan(scatx_corrected)] # otherwise raw pmt
         # Keep gain setting
-        gain = np.ones((len(raw), 1)) * 3
-        gain[np.isnan(raw[:, self.idx_SigOn3])] = 2
-        gain[np.isnan(raw[:, self.idx_SigOn2])] = 1
+        if self.firmware_version == 1:
+            gain = np.ones((len(raw), 1)) * 3
+            gain[np.isnan(raw[:, self.idx_SigOn3])] = 2
+            gain[np.isnan(raw[:, self.idx_SigOn2])] = 1
+        else:
+            # TODO Check if method is correct based on documentation
+            gain = np.ones((len(raw), 1)) * 3
+            gain[raw[:, self.idx_ChSaturated] == 3] = 2
+            gain[raw[:, self.idx_ChSaturated] == 2] = 1
         # Calculate beta
         uwl = np.unique(wl)
         # mu = pchip_interpolate(self.wavelength, self.mu, uwl)  # Optimized as no need of interpolation as same wavelength as calibration

inlinino/instruments/lisst.py

@@ -68,6 +68,7 @@ def setup(self, cfg):
         self.spectrum_plot_x_values = [np.log10(self._parser.angles)]
         # Update Active Timeseries Variables
         self.widget_active_timeseries_variables_names = ['beta(%.5f)' % x for x in self._parser.angles]
+        self.widget_active_timeseries_variables_selected = []
         self.active_timeseries_angles = np.zeros(len(self._parser.angles), dtype=bool)
         for theta in [0.08, 0.32, 1.28, 5.12]:
             channel_name = 'beta(%.5f)' % self._parser.angles[np.argmin(np.abs(self._parser.angles - theta))]

inlinino/instruments/satlantic.py

@@ -133,6 +133,7 @@ def setup(self, cfg):
         # TODO if no core variables disable spectrum plot widget
         # Update Active Timeseries Variables
         self.widget_active_timeseries_variables_names = []
+        self.widget_active_timeseries_variables_selected = []
         self.active_timeseries_variables = []
         for head, cal in self._parser.cal.items():
             self.widget_active_timeseries_variables_names += [f'{head}_{k}' for k in cal.key if k not in self.KEYS_TO_IGNORE]