From aef732153f32b520ee352d25832a2ddebe288b89 Mon Sep 17 00:00:00 2001 From: Nils Date: Tue, 22 Oct 2024 10:00:15 -0400 Subject: [PATCH] Add support for white space as separator - add missing setup_hyperbb.ui ressource - refactor hyperbb "firmware version" to "data format" - add support for white space as terminator or separator for generic instrument in GUI (already supported by the rest of the code) --- inlinino/__init__.py | 2 +- inlinino/gui.py | 31 +++++++------- inlinino/instruments/hyperbb.py | 63 ++++++++++++++++------------- inlinino/resources/setup_hyperbb.ui | 43 +++++++++++++++++++- 4 files changed, 95 insertions(+), 44 deletions(-) diff --git a/inlinino/__init__.py b/inlinino/__init__.py index d698373..bc80656 100644 --- a/inlinino/__init__.py +++ b/inlinino/__init__.py @@ -8,7 +8,7 @@ import numpy as np -__version__ = '2.9.14.delta' +__version__ = '2.9.14.epsilon' # Setup Logger logging.basicConfig(level=logging.DEBUG) diff --git a/inlinino/gui.py b/inlinino/gui.py index 78734c9..d25db8e 100644 --- a/inlinino/gui.py +++ b/inlinino/gui.py @@ -845,22 +845,25 @@ def act_save(self): if f in ['combobox_interface', 'combobox_model', *[f'combobox_relay{i}_mode' for i in range(4)]]: self.cfg[field_name] = self.__dict__[f].currentText() elif field_prefix in ['le', 'sb', 'dsb']: - value = getattr(self, f).text().strip() - if not field_optional and not value: - empty_fields.append(field_pretty_name) - if self.cfg['module'] == 'dataq' and field_name.startswith('variable'): - self.cfg[field_name] = [] - continue - # Apply special formatting to specific variables try: - if 'variable_' in field_name: + value = getattr(self, f).text() + # Pre-format value (needed here to handle special characters of some fields) + if field_name in ['terminator', 'separator']: + # Space are allowed as terminator or separator + value = value.encode(self.ENCODING).decode('unicode_escape').encode(self.ENCODING) + else: + value = value.strip() + # Check if field is optional, add to warning list otherwise + if not field_optional and not value: + empty_fields.append(field_pretty_name) + if self.cfg['module'] == 'dataq' and field_name.startswith('variable'): + self.cfg[field_name] = [] + continue + # Apply special formatting + if 'variable_' in field_name: # Generic Instrument: Variable Names, Units, Columns, Types, Precision value = [v.strip() for v in value.split(',')] if 'variable_columns' in field_name: value = [int(x) for x in value] - elif field_name in ['terminator', 'separator']: - # if len(value) > 3 and (value[:1] == "b'" and value[-1] == "'"): - # value = bytes(value[2:-1], 'ascii') - value = value.strip().encode(self.ENCODING).decode('unicode_escape').encode(self.ENCODING) elif field_prefix == 'sb': # SpinBox # a spinbox will contain either an int or float try: @@ -869,8 +872,6 @@ def act_save(self): value = float(value) elif field_prefix == 'dsb': # DoubleSpinBox value = float(value) - else: - value.strip() except: self.notification('Unable to parse special variable: ' + field_pretty_name, sys.exc_info()[0]) return @@ -901,7 +902,7 @@ def act_save(self): try: value = float(value) except ValueError: - pass + pass # Type is string self.cfg[field_name] = value elif field_prefix == 'cb': self.cfg[field_name] = getattr(self, f).isChecked() diff --git a/inlinino/instruments/hyperbb.py b/inlinino/instruments/hyperbb.py index 7159321..647e292 100644 --- a/inlinino/instruments/hyperbb.py +++ b/inlinino/instruments/hyperbb.py @@ -50,9 +50,9 @@ def setup(self, cfg): raise ValueError('Missing calibration plaque file (*.mat)') if 'temperature_file' not in cfg.keys(): raise ValueError('Missing calibration temperature file (*.mat)') - if 'firmware_version' not in cfg.keys(): - cfg['firmware_version'] = 1 - self._parser = HyperBBParser(cfg['plaque_file'], cfg['temperature_file'], cfg['firmware_version']) + if 'data_format' not in cfg.keys(): + cfg['data_format'] = 'advanced' + self._parser = HyperBBParser(cfg['plaque_file'], cfg['temperature_file'], cfg['data_format']) self.signal_reconstructed = np.empty(len(self._parser.wavelength)) * np.nan # Overload cfg with received data prod_var_names = ['beta_u', 'bb'] @@ -84,8 +84,8 @@ def parse(self, packet): self.signal.packet_corrupted.emit() if self.invalid_packet_alarm_triggered is False: self.invalid_packet_alarm_triggered = True - self.logger.warning('Unable to parse frame. Check firmware version.') - self.signal.alarm_custom.emit('Unable to parse frame.', 'Check HyperBB firmware version in "Setup".') + self.logger.warning('Unable to parse frame. Check data format.') + self.signal.alarm_custom.emit('Unable to parse frame.', 'Check HyperBB data format in "Setup".') return data def handle_data(self, raw, timestamp): @@ -137,11 +137,22 @@ def update_active_timeseries_variables(self, name, state): ['beta(%d)' % wl for wl in self._parser.wavelength[self.active_timeseries_wavelength]] +ADVANCED_DATA_FORMAT = 1 +LIGHT_DATA_FORMAT = 2 + class HyperBBParser(): - def __init__(self, plaque_cal_file, temperature_cal_file, firmware_version=1): + def __init__(self, plaque_cal_file, temperature_cal_file, data_format='advanced'): # Frame Parser - self.firmware_version = firmware_version - if firmware_version == 1: + if data_format.lower() == 'advanced': + self.data_format = ADVANCED_DATA_FORMAT + elif data_format.lower() == 'light': + self.data_format = LIGHT_DATA_FORMAT + else: + raise ValueError('Data format not recognized.') + if self.data_format == ADVANCED_DATA_FORMAT: + # The advanced output contains extra parameters: + # - The standard deviation can be used as a proxy for particle size. + # - The stepper position can be used to determine wavelength registration in case of instrument issues. self.FRAME_VARIABLES = ['ScanIdx', 'DataIdx', 'Date', 'Time', 'StepPos', 'wl', 'LedPwr', 'PmtGain', 'NetSig1', 'SigOn1', 'SigOn1Std', 'RefOn', 'RefOnStd', 'SigOff1', 'SigOff1Std', 'RefOff', 'RefOffStd', 'SigOn2', 'SigOn2Std', 'SigOn3', 'SigOn3Std', 'SigOff2', 'SigOff2Std', @@ -157,17 +168,13 @@ def __init__(self, plaque_cal_file, temperature_cal_file, firmware_version=1): self.FRAME_PRECISIONS = ['%s'] * len(self.FRAME_VARIABLES) for x in self.FRAME_VARIABLES: setattr(self, f'idx_{x}', self.FRAME_VARIABLES.index(x)) - elif firmware_version == 2: + elif self.data_format == LIGHT_DATA_FORMAT: self.FRAME_VARIABLES = ['ScanIdx', 'Date', 'Time', 'wl', 'PmtGain', 'NetRef', 'NetSig1', 'NetSig2', 'NetSig3', 'LedTemp', 'WaterTemp', 'Depth', 'SupplyVolt', 'ChSaturated'] self.FRAME_TYPES = [int, str, str, int, int, float, float, float, float, float, - float, float, float, float, float] - # FRAME_PRECISIONS = ['%d', '%d', '%s', '%s', '%d', '%d', '%d', '%d', '%d', - # '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', - # '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', - # '%.1f', '%.1f', '%.2f', '%.2f', '%.2f', '%d', '%d'] + float, float, float, float, int] self.FRAME_PRECISIONS = ['%s'] * len(self.FRAME_VARIABLES) for x in self.FRAME_VARIABLES: setattr(self, f'idx_{x}', self.FRAME_VARIABLES.index(x)) @@ -261,7 +268,7 @@ def calibrate(self, raw): raw = np.delete(raw, sel, axis=0) # Shortcuts wl = raw[:, self.idx_wl] - if self.firmware_version == 1: + if self.data_format == ADVANCED_DATA_FORMAT: # Remove saturated reading raw[raw[:, self.idx_SigOn1] > self.saturation_level, self.idx_SigOn1] = np.nan raw[raw[:, self.idx_SigOn2] > self.saturation_level, self.idx_SigOn2] = np.nan @@ -278,12 +285,24 @@ def calibrate(self, raw): scat1 = raw[:, self.idx_NetSig1] / net_ref scat2 = net_sig2 / net_ref scat3 = net_sig3 / net_ref - else: # Assume firmware 2 + # Keep gain setting + gain = np.ones((len(raw), 1)) * 3 + gain[np.isnan(raw[:, self.idx_SigOn3])] = 2 + gain[np.isnan(raw[:, self.idx_SigOn2])] = 1 + gain[np.isnan(raw[:, self.idx_SigOn1])] = 0 # All signals saturated + else: # Light Format net_ref_zero_flag = np.any(raw[:, self.idx_NetRef] == 0) - # TODO Check if need to flag saturated (ChSaturated) + raw[raw[:, self.idx_ChSaturated] == 1, self.idx_NetSig1] = np.nan + raw[(0 < raw[:, self.idx_ChSaturated]) & (raw[:, self.idx_ChSaturated] <= 2), self.idx_NetSig2] = np.nan + raw[(0 < raw[:, self.idx_ChSaturated]) & (raw[:, self.idx_ChSaturated] <= 3), self.idx_NetSig3] = np.nan scat1 = raw[:, self.idx_NetSig1] / raw[:, self.idx_NetRef] scat2 = raw[:, self.idx_NetSig2] / raw[:, self.idx_NetRef] scat3 = raw[:, self.idx_NetSig3] / raw[:, self.idx_NetRef] + # Keep Gain setting + gain = np.ones((len(raw), 1)) * 3 + gain[raw[:, self.idx_ChSaturated] == 3] = 2 + gain[raw[:, self.idx_ChSaturated] == 2] = 1 + gain[raw[:, self.idx_ChSaturated] == 1] = 0 # All signals saturated # Subtract dark offset scat1_dark_removed = scat1 - self.f_dark_cal_scat_1(raw[:, self.idx_PmtGain], wl) scat2_dark_removed = scat2 - self.f_dark_cal_scat_2(raw[:, self.idx_PmtGain], wl) @@ -302,16 +321,6 @@ def calibrate(self, raw): scatx_corrected = scat3_t_corrected # default is high gain scatx_corrected[np.isnan(scatx_corrected)] = scat2_t_corrected[np.isnan(scatx_corrected)] # otherwise low gain scatx_corrected[np.isnan(scatx_corrected)] = scat1_t_corrected[np.isnan(scatx_corrected)] # otherwise raw pmt - # Keep gain setting - if self.firmware_version == 1: - gain = np.ones((len(raw), 1)) * 3 - gain[np.isnan(raw[:, self.idx_SigOn3])] = 2 - gain[np.isnan(raw[:, self.idx_SigOn2])] = 1 - else: - # TODO Check if method is correct based on documentation - gain = np.ones((len(raw), 1)) * 3 - gain[raw[:, self.idx_ChSaturated] == 3] = 2 - gain[raw[:, self.idx_ChSaturated] == 2] = 1 # Calculate beta uwl = np.unique(wl) # mu = pchip_interpolate(self.wavelength, self.mu, uwl) # Optimized as no need of interpolation as same wavelength as calibration diff --git a/inlinino/resources/setup_hyperbb.ui b/inlinino/resources/setup_hyperbb.ui index 7ca54b1..4ce610f 100644 --- a/inlinino/resources/setup_hyperbb.ui +++ b/inlinino/resources/setup_hyperbb.ui @@ -10,7 +10,7 @@ 0 0 515 - 340 + 366 @@ -261,6 +261,47 @@ + + + + Data Format + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + + + + light + + + + + advanced + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + +