Add support for white space as separator

- add missing setup_hyperbb.ui ressource - refactor hyperbb "firmware version" to "data format" - add support for white space as terminator or separator for generic instrument in GUI (already supported by the rest of the code)
OceanOptics · Oct 22, 2024 · aef7321 · aef7321
1 parent c645262
commit aef7321
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 44 deletions.
diff --git a/inlinino/__init__.py b/inlinino/__init__.py
@@ -8,7 +8,7 @@
 import numpy as np
 
 
-__version__ = '2.9.14.delta'
+__version__ = '2.9.14.epsilon'
 
 # Setup Logger
 logging.basicConfig(level=logging.DEBUG)

diff --git a/inlinino/gui.py b/inlinino/gui.py
@@ -845,22 +845,25 @@ def act_save(self):
             if f in ['combobox_interface', 'combobox_model', *[f'combobox_relay{i}_mode' for i in range(4)]]:
                 self.cfg[field_name] = self.__dict__[f].currentText()
             elif field_prefix in ['le', 'sb', 'dsb']:
-                value = getattr(self, f).text().strip()
-                if not field_optional and not value:
-                    empty_fields.append(field_pretty_name)
-                    if self.cfg['module'] == 'dataq' and field_name.startswith('variable'):
-                        self.cfg[field_name] = []
-                    continue
-                # Apply special formatting to specific variables
                 try:
-                    if 'variable_' in field_name:
+                    value = getattr(self, f).text()
+                    # Pre-format value (needed here to handle special characters of some fields)
+                    if field_name in ['terminator', 'separator']:
+                        # Space are allowed as terminator or separator
+                        value = value.encode(self.ENCODING).decode('unicode_escape').encode(self.ENCODING)
+                    else:
+                        value = value.strip()
+                    # Check if field is optional, add to warning list otherwise
+                    if not field_optional and not value:
+                        empty_fields.append(field_pretty_name)
+                        if self.cfg['module'] == 'dataq' and field_name.startswith('variable'):
+                            self.cfg[field_name] = []
+                        continue
+                    # Apply special formatting
+                    if 'variable_' in field_name:  # Generic Instrument: Variable Names, Units, Columns, Types, Precision
                         value = [v.strip() for v in value.split(',')]
                         if 'variable_columns' in field_name:
                             value = [int(x) for x in value]
-                    elif field_name in ['terminator', 'separator']:
-                        # if len(value) > 3 and (value[:1] == "b'" and value[-1] == "'"):
-                        #     value = bytes(value[2:-1], 'ascii')
-                        value = value.strip().encode(self.ENCODING).decode('unicode_escape').encode(self.ENCODING)
                     elif field_prefix == 'sb':  # SpinBox
                         # a spinbox will contain either an int or float
                         try:
@@ -869,8 +872,6 @@ def act_save(self):
                             value = float(value)
                     elif field_prefix == 'dsb':  # DoubleSpinBox
                         value = float(value)
-                    else:
-                        value.strip()
                 except:
                     self.notification('Unable to parse special variable: ' + field_pretty_name, sys.exc_info()[0])
                     return
@@ -901,7 +902,7 @@ def act_save(self):
                         try:
                             value = float(value)
                         except ValueError:
-                            pass
+                            pass  # Type is string
                     self.cfg[field_name] = value
             elif field_prefix == 'cb':
                 self.cfg[field_name] = getattr(self, f).isChecked()

diff --git a/inlinino/instruments/hyperbb.py b/inlinino/instruments/hyperbb.py
@@ -50,9 +50,9 @@ def setup(self, cfg):
             raise ValueError('Missing calibration plaque file (*.mat)')
         if 'temperature_file' not in cfg.keys():
             raise ValueError('Missing calibration temperature file (*.mat)')
-        if 'firmware_version' not in cfg.keys():
-            cfg['firmware_version'] = 1
-        self._parser = HyperBBParser(cfg['plaque_file'], cfg['temperature_file'], cfg['firmware_version'])
+        if 'data_format' not in cfg.keys():
+            cfg['data_format'] = 'advanced'
+        self._parser = HyperBBParser(cfg['plaque_file'], cfg['temperature_file'], cfg['data_format'])
         self.signal_reconstructed = np.empty(len(self._parser.wavelength)) * np.nan
         # Overload cfg with received data
         prod_var_names = ['beta_u', 'bb']
@@ -84,8 +84,8 @@ def parse(self, packet):
             self.signal.packet_corrupted.emit()
             if self.invalid_packet_alarm_triggered is False:
                 self.invalid_packet_alarm_triggered = True
-                self.logger.warning('Unable to parse frame. Check firmware version.')
-                self.signal.alarm_custom.emit('Unable to parse frame.', 'Check HyperBB firmware version in "Setup".')
+                self.logger.warning('Unable to parse frame. Check data format.')
+                self.signal.alarm_custom.emit('Unable to parse frame.', 'Check HyperBB data format in "Setup".')
         return data
 
     def handle_data(self, raw, timestamp):
@@ -137,11 +137,22 @@ def update_active_timeseries_variables(self, name, state):
             ['beta(%d)' % wl for wl in self._parser.wavelength[self.active_timeseries_wavelength]]
 
 
+ADVANCED_DATA_FORMAT = 1
+LIGHT_DATA_FORMAT = 2
+
 class HyperBBParser():
-    def __init__(self, plaque_cal_file, temperature_cal_file, firmware_version=1):
+    def __init__(self, plaque_cal_file, temperature_cal_file, data_format='advanced'):
         # Frame Parser
-        self.firmware_version = firmware_version
-        if firmware_version == 1:
+        if data_format.lower() == 'advanced':
+            self.data_format = ADVANCED_DATA_FORMAT
+        elif data_format.lower() == 'light':
+            self.data_format = LIGHT_DATA_FORMAT
+        else:
+            raise ValueError('Data format not recognized.')
+        if self.data_format == ADVANCED_DATA_FORMAT:
+            # The advanced output contains extra parameters:
+            #     - The standard deviation can be used as a proxy for particle size.
+            #     - The stepper position can be used to determine wavelength registration in case of instrument issues.
             self.FRAME_VARIABLES = ['ScanIdx', 'DataIdx', 'Date', 'Time', 'StepPos', 'wl', 'LedPwr', 'PmtGain', 'NetSig1',
                                    'SigOn1', 'SigOn1Std', 'RefOn', 'RefOnStd', 'SigOff1', 'SigOff1Std', 'RefOff',
                                    'RefOffStd', 'SigOn2', 'SigOn2Std', 'SigOn3', 'SigOn3Std', 'SigOff2', 'SigOff2Std',
@@ -157,17 +168,13 @@ def __init__(self, plaque_cal_file, temperature_cal_file, firmware_version=1):
             self.FRAME_PRECISIONS = ['%s'] * len(self.FRAME_VARIABLES)
             for x in self.FRAME_VARIABLES:
                 setattr(self, f'idx_{x}', self.FRAME_VARIABLES.index(x))
-        elif firmware_version == 2:
+        elif self.data_format == LIGHT_DATA_FORMAT:
             self.FRAME_VARIABLES = ['ScanIdx', 'Date', 'Time', 'wl', 'PmtGain',
                                    'NetRef', 'NetSig1', 'NetSig2', 'NetSig3',
                                    'LedTemp', 'WaterTemp', 'Depth', 'SupplyVolt', 'ChSaturated']
             self.FRAME_TYPES = [int, str, str, int, int,
                                 float, float, float, float, float,
-                                float, float, float, float, float]
-            # FRAME_PRECISIONS = ['%d', '%d', '%s', '%s', '%d', '%d', '%d', '%d', '%d',
-            #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
-            #                    '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f',
-            #                    '%.1f', '%.1f', '%.2f', '%.2f', '%.2f', '%d', '%d']
+                                float, float, float, float, int]
             self.FRAME_PRECISIONS = ['%s'] * len(self.FRAME_VARIABLES)
             for x in self.FRAME_VARIABLES:
                 setattr(self, f'idx_{x}', self.FRAME_VARIABLES.index(x))
@@ -261,7 +268,7 @@ def calibrate(self, raw):
                     raw = np.delete(raw, sel, axis=0)
         # Shortcuts
         wl = raw[:, self.idx_wl]
-        if self.firmware_version == 1:
+        if self.data_format == ADVANCED_DATA_FORMAT:
             # Remove saturated reading
             raw[raw[:, self.idx_SigOn1] > self.saturation_level, self.idx_SigOn1] = np.nan
             raw[raw[:, self.idx_SigOn2] > self.saturation_level, self.idx_SigOn2] = np.nan
@@ -278,12 +285,24 @@ def calibrate(self, raw):
             scat1 = raw[:, self.idx_NetSig1] / net_ref
             scat2 = net_sig2 / net_ref
             scat3 = net_sig3 / net_ref
-        else:  # Assume firmware 2
+            # Keep gain setting
+            gain = np.ones((len(raw), 1)) * 3
+            gain[np.isnan(raw[:, self.idx_SigOn3])] = 2
+            gain[np.isnan(raw[:, self.idx_SigOn2])] = 1
+            gain[np.isnan(raw[:, self.idx_SigOn1])] = 0  # All signals saturated
+        else:  # Light Format
             net_ref_zero_flag = np.any(raw[:, self.idx_NetRef] == 0)
-            # TODO Check if need to flag saturated (ChSaturated)
+            raw[raw[:, self.idx_ChSaturated] == 1, self.idx_NetSig1] = np.nan
+            raw[(0 < raw[:, self.idx_ChSaturated]) & (raw[:, self.idx_ChSaturated] <= 2), self.idx_NetSig2] = np.nan
+            raw[(0 < raw[:, self.idx_ChSaturated]) & (raw[:, self.idx_ChSaturated] <= 3), self.idx_NetSig3] = np.nan
             scat1 = raw[:, self.idx_NetSig1] / raw[:, self.idx_NetRef]
             scat2 = raw[:, self.idx_NetSig2] / raw[:, self.idx_NetRef]
             scat3 = raw[:, self.idx_NetSig3] / raw[:, self.idx_NetRef]
+            # Keep Gain setting
+            gain = np.ones((len(raw), 1)) * 3
+            gain[raw[:, self.idx_ChSaturated] == 3] = 2
+            gain[raw[:, self.idx_ChSaturated] == 2] = 1
+            gain[raw[:, self.idx_ChSaturated] == 1] = 0  # All signals saturated
         # Subtract dark offset
         scat1_dark_removed = scat1 - self.f_dark_cal_scat_1(raw[:, self.idx_PmtGain], wl)
         scat2_dark_removed = scat2 - self.f_dark_cal_scat_2(raw[:, self.idx_PmtGain], wl)
@@ -302,16 +321,6 @@ def calibrate(self, raw):
         scatx_corrected = scat3_t_corrected  # default is high gain
         scatx_corrected[np.isnan(scatx_corrected)] = scat2_t_corrected[np.isnan(scatx_corrected)] # otherwise low gain
         scatx_corrected[np.isnan(scatx_corrected)] = scat1_t_corrected[np.isnan(scatx_corrected)] # otherwise raw pmt
-        # Keep gain setting
-        if self.firmware_version == 1:
-            gain = np.ones((len(raw), 1)) * 3
-            gain[np.isnan(raw[:, self.idx_SigOn3])] = 2
-            gain[np.isnan(raw[:, self.idx_SigOn2])] = 1
-        else:
-            # TODO Check if method is correct based on documentation
-            gain = np.ones((len(raw), 1)) * 3
-            gain[raw[:, self.idx_ChSaturated] == 3] = 2
-            gain[raw[:, self.idx_ChSaturated] == 2] = 1
         # Calculate beta
         uwl = np.unique(wl)
         # mu = pchip_interpolate(self.wavelength, self.mu, uwl)  # Optimized as no need of interpolation as same wavelength as calibration

diff --git a/inlinino/resources/setup_hyperbb.ui b/inlinino/resources/setup_hyperbb.ui
@@ -10,7 +10,7 @@
     <x>0</x>
     <y>0</y>
     <width>515</width>
-    <height>340</height>
+    <height>366</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -261,6 +261,47 @@
         </item>
        </layout>
       </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="label_5">
+        <property name="text">
+         <string>Data Format</string>
+        </property>
+        <property name="alignment">
+         <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <layout class="QHBoxLayout" name="horizontalLayout_3">
+        <item>
+         <widget class="QComboBox" name="combobox_data_format">
+          <item>
+           <property name="text">
+            <string>light</string>
+           </property>
+          </item>
+          <item>
+           <property name="text">
+            <string>advanced</string>
+           </property>
+          </item>
+         </widget>
+        </item>
+        <item>
+         <spacer name="horizontalSpacer">
+          <property name="orientation">
+           <enum>Qt::Horizontal</enum>
+          </property>
+          <property name="sizeHint" stdset="0">
+           <size>
+            <width>40</width>
+            <height>20</height>
+           </size>
+          </property>
+         </spacer>
+        </item>
+       </layout>
+      </item>
      </layout>
     </widget>
    </item>