From ed0ee2c275caf04ff880da34c65ff75a9d49aa42 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 7 Dec 2023 12:27:14 -0700 Subject: [PATCH 01/16] Use default gain of 0 and update calibration handling to allow no calibration parameters. --- scos_actions/calibration/calibration.py | 15 +-- .../configs/sensor_calibration_example.json | 98 ------------------- .../configs/sigan_calibration_example.json | 98 ------------------- scos_actions/hardware/sigan_iface.py | 2 +- scos_actions/metadata/utils.py | 2 +- scos_actions/settings.py | 8 +- 6 files changed, 14 insertions(+), 209 deletions(-) delete mode 100644 scos_actions/configs/sensor_calibration_example.json delete mode 100644 scos_actions/configs/sigan_calibration_example.json diff --git a/scos_actions/calibration/calibration.py b/scos_actions/calibration/calibration.py index d028d004..4f14a4c4 100644 --- a/scos_actions/calibration/calibration.py +++ b/scos_actions/calibration/calibration.py @@ -38,14 +38,15 @@ def get_calibration_dict(self, cal_params: List[Union[float, int, bool]]) -> dic then the input to this method could be ``["15360000.0", "40"]``. :return: The calibration data corresponding to the input parameter values. """ - # Check if the sample rate was calibrated + cal_data = self.calibration_data - # raise Exception(self) - for i, setting_value in enumerate(cal_params): - setting = self.calibration_parameters[i] - logger.debug(f"Looking up calibration for {setting} at {setting_value}") - cal_data = filter_by_parameter(cal_data, setting_value) - logger.debug(f"Got calibration data: {cal_data}") + if len(cal_params) > 0: + for i, setting_value in enumerate(cal_params): + setting = self.calibration_parameters[i] + logger.debug(f"Looking up calibration for {setting} at {setting_value}") + cal_data = filter_by_parameter(cal_data, setting_value) + logger.debug(f"Got calibration data: {cal_data}") + return cal_data def update( diff --git a/scos_actions/configs/sensor_calibration_example.json b/scos_actions/configs/sensor_calibration_example.json deleted file mode 100644 index 669c7445..00000000 --- a/scos_actions/configs/sensor_calibration_example.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "calibration_data": { - "10000000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - }, - "15360000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - } - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [ - "sample_rate", - "frequency", - "gain" - ], - "clock_rate_lookup_by_sample_rate": [ - { - "clock_frequency": 40000000.0, - "sample_rate": 10000000.0 - }, - { - "clock_frequency": 30720000.0, - "sample_rate": 15360000.0 - } - ], - "sensor_uid": "EXAMPLE", - "calibration_reference": "noise source output" -} diff --git a/scos_actions/configs/sigan_calibration_example.json b/scos_actions/configs/sigan_calibration_example.json deleted file mode 100644 index 669c7445..00000000 --- a/scos_actions/configs/sigan_calibration_example.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "calibration_data": { - "10000000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - }, - "15360000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - } - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [ - "sample_rate", - "frequency", - "gain" - ], - "clock_rate_lookup_by_sample_rate": [ - { - "clock_frequency": 40000000.0, - "sample_rate": 10000000.0 - }, - { - "clock_frequency": 30720000.0, - "sample_rate": 15360000.0 - } - ], - "sensor_uid": "EXAMPLE", - "calibration_reference": "noise source output" -} diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index a12c1d67..e3b927e3 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -33,7 +33,7 @@ def __init__(self): # Define the default calibration dicts self.DEFAULT_SIGAN_CALIBRATION = { "datetime": get_datetime_str_now(), - "gain": None, # Defaults to gain setting + "gain": 0, # Defaults to gain setting "enbw": None, # Defaults to sample rate "noise_figure": 0, "1db_compression_point": 100, diff --git a/scos_actions/metadata/utils.py b/scos_actions/metadata/utils.py index 6253049a..f2704d65 100644 --- a/scos_actions/metadata/utils.py +++ b/scos_actions/metadata/utils.py @@ -35,7 +35,7 @@ def construct_geojson_point( def _enc_hook(obj: Any) -> Any: #While isinstance is recommended, it was causing a #Recurrsion error and I don't think we have to worry - #about subytpes here. + #about subytpes here. if type(obj) == np.float64: return float(obj) elif type(obj) == np.bool_: diff --git a/scos_actions/settings.py b/scos_actions/settings.py index e8236f75..dee2eb73 100644 --- a/scos_actions/settings.py +++ b/scos_actions/settings.py @@ -13,16 +13,16 @@ # set sigan_calibration file and sensor_calibration_file if not settings.configured or not hasattr(settings, "SIGAN_CALIBRATION_FILE"): - logger.warning("Using default sigan cal file.") - SIGAN_CALIBRATION_FILE = CONFIG_DIR / "sigan_calibration_example.json" + logger.warning("Using default sigan calibration.") + SIGAN_CALIBRATION_FILE = "" sigan_calibration = None else: SIGAN_CALIBRATION_FILE = Path(settings.SIGAN_CALIBRATION_FILE) logger.debug(f"SCOS_ACTIONS: SIGAN_CALIBRATION_FILE: {SIGAN_CALIBRATION_FILE}") if not settings.configured or not hasattr(settings, "SENSOR_CALIBRATION_FILE"): - logger.warning("Using default sensor cal file.") - SENSOR_CALIBRATION_FILE = CONFIG_DIR / "sensor_calibration_example.json" + logger.warning("Using default calibration.") + SENSOR_CALIBRATION_FILE = "" sensor_calibration = None else: SENSOR_CALIBRATION_FILE = Path(settings.SENSOR_CALIBRATION_FILE) From 06b7cfd8c716eebf10240c7a4c6b1031964f47d3 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Thu, 7 Dec 2023 14:23:54 -0700 Subject: [PATCH 02/16] Increment version to 7.1.0 --- scos_actions/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index b378fc6c..54ccb5d2 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -1 +1 @@ -__version__ = "7.0.0" +__version__ = "7.1.0" From 4fd7152017668c6f6636c19ee6035ae025428e1e Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 11 Dec 2023 08:56:07 -0700 Subject: [PATCH 03/16] Set gain to 0 and noise figure to None in default calibration data. Add test for default sensor calibration data. --- scos_actions/calibration/tests/test_calibration.py | 6 ------ scos_actions/hardware/sigan_iface.py | 4 ++-- scos_actions/hardware/tests/test_sigan.py | 10 ++++++++++ scos_actions/metadata/utils.py | 8 ++++---- 4 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 scos_actions/hardware/tests/test_sigan.py diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py index 8d07df36..75fc1eb7 100644 --- a/scos_actions/calibration/tests/test_calibration.py +++ b/scos_actions/calibration/tests/test_calibration.py @@ -310,12 +310,6 @@ def test_update(self): assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0 assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0 - def test_default_sensor_cal(self): - assert sensor_calibration is not None - - def test_default_sigan_cal_location(self): - assert sigan_calibration is not None - def test_filter_by_paramter_integer(self): calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}} filtered_data = filter_by_parameter(calibrations, 200) diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index e3b927e3..218c28c8 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -35,14 +35,14 @@ def __init__(self): "datetime": get_datetime_str_now(), "gain": 0, # Defaults to gain setting "enbw": None, # Defaults to sample rate - "noise_figure": 0, + "noise_figure": None, "1db_compression_point": 100, "temperature": 26.85, } self.DEFAULT_SENSOR_CALIBRATION = { "datetime": get_datetime_str_now(), - "gain": None, # Defaults to sigan gain + "gain": 0, # Defaults to sigan gain "enbw": None, # Defaults to sigan enbw "noise_figure": None, # Defaults to sigan noise figure "1db_compression_point": None, # Defaults to sigan compression + preselector gain diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py new file mode 100644 index 00000000..914b569e --- /dev/null +++ b/scos_actions/hardware/tests/test_sigan.py @@ -0,0 +1,10 @@ +from environs import Env + +from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer + + +def test_sigan_default_cal(): + sigan = MockSignalAnalyzer() + sigan.recompute_sensor_calibration_data([]) + sensor_cal = sigan.sensor_calibration_data + assert sensor_cal["gain"] == 0 \ No newline at end of file diff --git a/scos_actions/metadata/utils.py b/scos_actions/metadata/utils.py index f2704d65..d814249f 100644 --- a/scos_actions/metadata/utils.py +++ b/scos_actions/metadata/utils.py @@ -33,10 +33,10 @@ def construct_geojson_point( def _enc_hook(obj: Any) -> Any: - #While isinstance is recommended, it was causing a - #Recurrsion error and I don't think we have to worry - #about subytpes here. - if type(obj) == np.float64: + # While isinstance is recommended, it was causing a + # Recurrsion error and I don't think we have to worry + # about subytpes here. + if type(obj) == np.float64: return float(obj) elif type(obj) == np.bool_: return bool(obj) From 37b61d2aff5316ad263b887aa73bc44edd4336cd Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 11 Dec 2023 10:12:13 -0700 Subject: [PATCH 04/16] remove unnecessary length check of cal params. --- scos_actions/calibration/calibration.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scos_actions/calibration/calibration.py b/scos_actions/calibration/calibration.py index 4f14a4c4..7514a379 100644 --- a/scos_actions/calibration/calibration.py +++ b/scos_actions/calibration/calibration.py @@ -40,12 +40,11 @@ def get_calibration_dict(self, cal_params: List[Union[float, int, bool]]) -> dic """ cal_data = self.calibration_data - if len(cal_params) > 0: - for i, setting_value in enumerate(cal_params): - setting = self.calibration_parameters[i] - logger.debug(f"Looking up calibration for {setting} at {setting_value}") - cal_data = filter_by_parameter(cal_data, setting_value) - logger.debug(f"Got calibration data: {cal_data}") + for i, setting_value in enumerate(cal_params): + setting = self.calibration_parameters[i] + logger.debug(f"Looking up calibration for {setting} at {setting_value}") + cal_data = filter_by_parameter(cal_data, setting_value) + logger.debug(f"Got calibration data: {cal_data}") return cal_data From b754cf7679303bd7bd6df45b454249f9cb272a2d Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 18 Dec 2023 08:26:18 -0700 Subject: [PATCH 05/16] Correct type hints in get_sensor_calibration and git_sigan_calibration. Add warnings during sigan and sensor cal loading. Add warning if using default calibration for sensor or sigan cal. Add sigan_calibration and sensor_calibration to base SignalAnalyzerInterface. --- scos_actions/calibration/__init__.py | 37 +++++++++++++++++------ scos_actions/hardware/mocks/mock_sigan.py | 35 +++++++++++++++++++++ scos_actions/hardware/sigan_iface.py | 32 ++++++-------------- 3 files changed, 72 insertions(+), 32 deletions(-) diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index 14326964..c9e94b54 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -1,13 +1,12 @@ import logging -from pathlib import Path - +from os import path from scos_actions.calibration.calibration import Calibration, load_from_json from scos_actions.settings import SENSOR_CALIBRATION_FILE, SIGAN_CALIBRATION_FILE logger = logging.getLogger(__name__) +DEFAULT_CALIBRATION_FILE="no_calibration.json" - -def get_sigan_calibration(sigan_cal_file: Path) -> Calibration: +def get_sigan_calibration(sigan_cal_file: str) -> Calibration: """ Load signal analyzer calibration data from file. @@ -16,6 +15,7 @@ def get_sigan_calibration(sigan_cal_file: Path) -> Calibration: :return: The signal analyzer ``Calibration`` object. """ try: + check_for_default_calibration(str, "Sigan") sigan_cal = load_from_json(sigan_cal_file) except Exception: sigan_cal = None @@ -23,7 +23,7 @@ def get_sigan_calibration(sigan_cal_file: Path) -> Calibration: return sigan_cal -def get_sensor_calibration(sensor_cal_file: Path) -> Calibration: +def get_sensor_calibration(sensor_cal_file: str) -> Calibration: """ Load sensor calibration data from file. @@ -32,16 +32,35 @@ def get_sensor_calibration(sensor_cal_file: Path) -> Calibration: :return: The sensor ``Calibration`` object. """ try: + check_for_default_calibration(sensor_cal_file, "Sensor") sensor_cal = load_from_json(sensor_cal_file) except Exception: sensor_cal = None logger.exception("Unable to load sensor calibration data, reverting to none") return sensor_cal +def check_for_default_calibration(cal_file_path: str, cal_type: str): + cal_file_name = path.basename(cal_file_path) + if cal_file_name == DEFAULT_CALIBRATION_FILE: + logger.warning(f"***************LOADING DEFAULT {cal_type} CALIBRATION***************") + +sensor_calibration = None +if SENSOR_CALIBRATION_FILE is None: + logger.warning("Sensor calibration file is None. Not loading calibration file.") +elif not path.exists(SENSOR_CALIBRATION_FILE): + logger.warning(SENSOR_CALIBRATION_FILE + " does not exist. Not loading sensor calibration file.") +else: + logger.debug(f"Loading sensor cal file: {SENSOR_CALIBRATION_FILE}") + sensor_calibration = get_sensor_calibration(SENSOR_CALIBRATION_FILE) + +sigan_calibration = None +if SIGAN_CALIBRATION_FILE is None: + logger.warning("Sigan calibration file is None. Not loading calibration file.") +elif not path.exists(SIGAN_CALIBRATION_FILE): + logger.warning(SIGAN_CALIBRATION_FILE + " does not exist. Not loading sigan calibration file.") +else: + logger.debug(f"Loading sigan cal file: {SIGAN_CALIBRATION_FILE}") + sigan_calibration = get_sigan_calibration(SIGAN_CALIBRATION_FILE) -logger.debug(f"Loading sensor cal file: {SENSOR_CALIBRATION_FILE}") -sensor_calibration = get_sensor_calibration(SENSOR_CALIBRATION_FILE) -logger.debug(f"Loading sigan cal file: {SIGAN_CALIBRATION_FILE}") -sigan_calibration = get_sigan_calibration(SIGAN_CALIBRATION_FILE) if sensor_calibration: logger.debug(f"Last sensor cal: {sensor_calibration.last_calibration_datetime}") diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py index 46b8f2ec..fa740105 100644 --- a/scos_actions/hardware/mocks/mock_sigan.py +++ b/scos_actions/hardware/mocks/mock_sigan.py @@ -26,6 +26,24 @@ class MockSignalAnalyzer(SignalAnalyzerInterface): def __init__(self, randomize_values=False): super().__init__() + # Define the default calibration dicts + self.DEFAULT_SIGAN_CALIBRATION = { + "datetime": get_datetime_str_now(), + "gain": 0, # Defaults to gain setting + "enbw": None, # Defaults to sample rate + "noise_figure": 0, + "1db_compression_point": 100, + "temperature": 26.85, + } + + self.DEFAULT_SENSOR_CALIBRATION = { + "datetime": get_datetime_str_now(), + "gain": 0, # Defaults to sigan gain + "enbw": None, # Defaults to sigan enbw + "noise_figure": None, # Defaults to sigan noise figure + "1db_compression_point": None, # Defaults to sigan compression + preselector gain + "temperature": 26.85, + } self.auto_dc_offset = False self._frequency = 700e6 self._sample_rate = 10e6 @@ -172,3 +190,20 @@ def last_calibration_time(self): def update_calibration(self, params): pass + + def recompute_sensor_calibration_data(self, cal_args: list) -> None: + if self.sensor_calibration is not None: + self.sensor_calibration_data.update( + self.sensor_calibration.get_calibration_dict(cal_args) + ) + else: + logger.warning("Sensor calibration does not exist.") + + def recompute_sigan_calibration_data(self, cal_args: list) -> None: + """Set the sigan calibration data based on the current tuning""" + if self.sigan_calibration is not None: + self.sigan_calibration_data.update( + self.sigan_calibration.get_calibration_dict(cal_args) + ) + else: + logger.warning("Sigan calibration does not exist.") diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index 218c28c8..eae3d7ab 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -30,26 +30,10 @@ class SignalAnalyzerInterface(ABC): def __init__(self): - # Define the default calibration dicts - self.DEFAULT_SIGAN_CALIBRATION = { - "datetime": get_datetime_str_now(), - "gain": 0, # Defaults to gain setting - "enbw": None, # Defaults to sample rate - "noise_figure": None, - "1db_compression_point": 100, - "temperature": 26.85, - } - - self.DEFAULT_SENSOR_CALIBRATION = { - "datetime": get_datetime_str_now(), - "gain": 0, # Defaults to sigan gain - "enbw": None, # Defaults to sigan enbw - "noise_figure": None, # Defaults to sigan noise figure - "1db_compression_point": None, # Defaults to sigan compression + preselector gain - "temperature": 26.85, - } - self.sensor_calibration_data = copy.deepcopy(self.DEFAULT_SENSOR_CALIBRATION) - self.sigan_calibration_data = copy.deepcopy(self.DEFAULT_SIGAN_CALIBRATION) + self.sensor_calibration_data = {} + self.sigan_calibration_data = {} + self.sensor_calibration = sensor_calibration + self.sigan_calibration = sigan_calibration @property def last_calibration_time(self) -> str: @@ -125,7 +109,7 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None: logger.info("Attempting to power cycle the signal analyzer and reconnect.") try: power_cycle_sigan() - except HardwareConfigurationException as hce: + except Exception as hce: logger.warning(f"Unable to power cycle sigan: {hce}") return try: @@ -141,19 +125,21 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None: return def recompute_sensor_calibration_data(self, cal_args: list) -> None: - self.sensor_calibration_data = self.DEFAULT_SENSOR_CALIBRATION.copy() if sensor_calibration is not None: self.sensor_calibration_data.update( sensor_calibration.get_calibration_dict(cal_args) ) + else: + logger.warning("Sensor calibration does not exist.") def recompute_sigan_calibration_data(self, cal_args: list) -> None: """Set the sigan calibration data based on the current tuning""" - self.sigan_calibration_data = self.DEFAULT_SIGAN_CALIBRATION.copy() if sigan_calibration is not None: self.sigan_calibration_data.update( sigan_calibration.get_calibration_dict(cal_args) ) + else: + logger.warning("Sigan calibration does not exist.") def get_status(self): try: From b548321d5263fab040cf28495d8a6c8a87306487 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 18 Dec 2023 08:28:35 -0700 Subject: [PATCH 06/16] run pre-commit. --- scos_actions/calibration/__init__.py | 19 +++++++++++++++---- scos_actions/hardware/tests/test_sigan.py | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index c9e94b54..b48b5cda 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -1,10 +1,12 @@ import logging from os import path + from scos_actions.calibration.calibration import Calibration, load_from_json from scos_actions.settings import SENSOR_CALIBRATION_FILE, SIGAN_CALIBRATION_FILE logger = logging.getLogger(__name__) -DEFAULT_CALIBRATION_FILE="no_calibration.json" +DEFAULT_CALIBRATION_FILE = "no_calibration.json" + def get_sigan_calibration(sigan_cal_file: str) -> Calibration: """ @@ -39,16 +41,23 @@ def get_sensor_calibration(sensor_cal_file: str) -> Calibration: logger.exception("Unable to load sensor calibration data, reverting to none") return sensor_cal + def check_for_default_calibration(cal_file_path: str, cal_type: str): cal_file_name = path.basename(cal_file_path) if cal_file_name == DEFAULT_CALIBRATION_FILE: - logger.warning(f"***************LOADING DEFAULT {cal_type} CALIBRATION***************") + logger.warning( + f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" + ) + sensor_calibration = None if SENSOR_CALIBRATION_FILE is None: logger.warning("Sensor calibration file is None. Not loading calibration file.") elif not path.exists(SENSOR_CALIBRATION_FILE): - logger.warning(SENSOR_CALIBRATION_FILE + " does not exist. Not loading sensor calibration file.") + logger.warning( + SENSOR_CALIBRATION_FILE + + " does not exist. Not loading sensor calibration file." + ) else: logger.debug(f"Loading sensor cal file: {SENSOR_CALIBRATION_FILE}") sensor_calibration = get_sensor_calibration(SENSOR_CALIBRATION_FILE) @@ -57,7 +66,9 @@ def check_for_default_calibration(cal_file_path: str, cal_type: str): if SIGAN_CALIBRATION_FILE is None: logger.warning("Sigan calibration file is None. Not loading calibration file.") elif not path.exists(SIGAN_CALIBRATION_FILE): - logger.warning(SIGAN_CALIBRATION_FILE + " does not exist. Not loading sigan calibration file.") + logger.warning( + SIGAN_CALIBRATION_FILE + " does not exist. Not loading sigan calibration file." + ) else: logger.debug(f"Loading sigan cal file: {SIGAN_CALIBRATION_FILE}") sigan_calibration = get_sigan_calibration(SIGAN_CALIBRATION_FILE) diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py index 914b569e..df980ecf 100644 --- a/scos_actions/hardware/tests/test_sigan.py +++ b/scos_actions/hardware/tests/test_sigan.py @@ -7,4 +7,4 @@ def test_sigan_default_cal(): sigan = MockSignalAnalyzer() sigan.recompute_sensor_calibration_data([]) sensor_cal = sigan.sensor_calibration_data - assert sensor_cal["gain"] == 0 \ No newline at end of file + assert sensor_cal["gain"] == 0 From 534b0ac8ee14d8bfd98cd928dbd743d74a2cce01 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 18 Dec 2023 08:45:32 -0700 Subject: [PATCH 07/16] Import the name of the default calibration file from scos-sensor. --- scos_actions/calibration/__init__.py | 7 +++++-- scos_actions/settings.py | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index b48b5cda..c55feb27 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -2,10 +2,13 @@ from os import path from scos_actions.calibration.calibration import Calibration, load_from_json -from scos_actions.settings import SENSOR_CALIBRATION_FILE, SIGAN_CALIBRATION_FILE +from scos_actions.settings import ( + DEFAULT_CALIBRATION_FILE, + SENSOR_CALIBRATION_FILE, + SIGAN_CALIBRATION_FILE, +) logger = logging.getLogger(__name__) -DEFAULT_CALIBRATION_FILE = "no_calibration.json" def get_sigan_calibration(sigan_cal_file: str) -> Calibration: diff --git a/scos_actions/settings.py b/scos_actions/settings.py index dee2eb73..3384102b 100644 --- a/scos_actions/settings.py +++ b/scos_actions/settings.py @@ -11,9 +11,14 @@ CONFIG_DIR = Path(__file__).parent.resolve() / "configs" ACTION_DEFINITIONS_DIR = CONFIG_DIR / "actions" +if not settings.configured or not hasattr(settings, "DEFAULT_CALIBRATION_FILE"): + DEFAULT_CALIBRATION_FILE = "" +else: + DEFAULT_CALIBRATION_FILE = settings.DEFAULT_CALIBRATION_FILE + # set sigan_calibration file and sensor_calibration_file if not settings.configured or not hasattr(settings, "SIGAN_CALIBRATION_FILE"): - logger.warning("Using default sigan calibration.") + logger.warning("Sigan calibration file is not defined.") SIGAN_CALIBRATION_FILE = "" sigan_calibration = None else: @@ -21,7 +26,7 @@ logger.debug(f"SCOS_ACTIONS: SIGAN_CALIBRATION_FILE: {SIGAN_CALIBRATION_FILE}") if not settings.configured or not hasattr(settings, "SENSOR_CALIBRATION_FILE"): - logger.warning("Using default calibration.") + logger.warning("Sensor calibration file is not defined.") SENSOR_CALIBRATION_FILE = "" sensor_calibration = None else: From 8480a37dff509bcb0e1acd98ec41a238e4cbd05f Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Mon, 18 Dec 2023 08:49:26 -0700 Subject: [PATCH 08/16] Keep SIGAN_CALIBRATION_FILE and SENSOR_CALIBRATION_FILE as strings. --- scos_actions/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scos_actions/settings.py b/scos_actions/settings.py index 3384102b..7b628801 100644 --- a/scos_actions/settings.py +++ b/scos_actions/settings.py @@ -22,7 +22,7 @@ SIGAN_CALIBRATION_FILE = "" sigan_calibration = None else: - SIGAN_CALIBRATION_FILE = Path(settings.SIGAN_CALIBRATION_FILE) + SIGAN_CALIBRATION_FILE = settings.SIGAN_CALIBRATION_FILE logger.debug(f"SCOS_ACTIONS: SIGAN_CALIBRATION_FILE: {SIGAN_CALIBRATION_FILE}") if not settings.configured or not hasattr(settings, "SENSOR_CALIBRATION_FILE"): @@ -30,7 +30,7 @@ SENSOR_CALIBRATION_FILE = "" sensor_calibration = None else: - SENSOR_CALIBRATION_FILE = Path(settings.SENSOR_CALIBRATION_FILE) + SENSOR_CALIBRATION_FILE = settings.SENSOR_CALIBRATION_FILE logger.debug(f"SCOS_ACTIONS: SENSOR_CALIBRATION_FILE: {SENSOR_CALIBRATION_FILE}") SWITCH_CONFIGS_DIR = env("SWITCH_CONFIGS_DIR", default=None) From 1a9a29b7e5d27d83c45ee21995121e823b13932b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 08:44:12 -0700 Subject: [PATCH 09/16] typo fix. --- scos_actions/calibration/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index c55feb27..be43daf0 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -20,7 +20,7 @@ def get_sigan_calibration(sigan_cal_file: str) -> Calibration: :return: The signal analyzer ``Calibration`` object. """ try: - check_for_default_calibration(str, "Sigan") + check_for_default_calibration(sigan_cal_file, "Sigan") sigan_cal = load_from_json(sigan_cal_file) except Exception: sigan_cal = None From 0489f1fcf0afa753226b5c3b6c368a5f9bcf08f6 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 09:27:05 -0700 Subject: [PATCH 10/16] Set sensor/sigan_calibration_data to empty dict when recomputing cal data. --- scos_actions/hardware/sigan_iface.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index eae3d7ab..ed1d5189 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -5,14 +5,8 @@ from scos_actions.calibration import sensor_calibration, sigan_calibration from scos_actions.capabilities import capabilities -from scos_actions.hardware.hardware_configuration_exception import ( - HardwareConfigurationException, -) from scos_actions.hardware.utils import power_cycle_sigan -from scos_actions.utils import ( - convert_string_to_millisecond_iso_format, - get_datetime_str_now, -) +from scos_actions.utils import convert_string_to_millisecond_iso_format logger = logging.getLogger(__name__) @@ -125,6 +119,7 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None: return def recompute_sensor_calibration_data(self, cal_args: list) -> None: + self.sensor_calibration_data = {} if sensor_calibration is not None: self.sensor_calibration_data.update( sensor_calibration.get_calibration_dict(cal_args) @@ -133,6 +128,7 @@ def recompute_sensor_calibration_data(self, cal_args: list) -> None: logger.warning("Sensor calibration does not exist.") def recompute_sigan_calibration_data(self, cal_args: list) -> None: + self.sigan_calibration_data = {} """Set the sigan calibration data based on the current tuning""" if sigan_calibration is not None: self.sigan_calibration_data.update( From c9ee0be7031179e845f1f03719a146f183a08934 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 09:38:54 -0700 Subject: [PATCH 11/16] use path in check for default cal file. --- scos_actions/calibration/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index be43daf0..7d2db429 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -46,8 +46,7 @@ def get_sensor_calibration(sensor_cal_file: str) -> Calibration: def check_for_default_calibration(cal_file_path: str, cal_type: str): - cal_file_name = path.basename(cal_file_path) - if cal_file_name == DEFAULT_CALIBRATION_FILE: + if cal_file_path == DEFAULT_CALIBRATION_FILE: logger.warning( f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" ) From 9f10d5d8e7d72c8719627cbfa26f566a1609bd78 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 09:56:11 -0700 Subject: [PATCH 12/16] set calibration parameters in calibration update if there are none. --- scos_actions/calibration/calibration.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scos_actions/calibration/calibration.py b/scos_actions/calibration/calibration.py index 7514a379..9d724af7 100644 --- a/scos_actions/calibration/calibration.py +++ b/scos_actions/calibration/calibration.py @@ -77,9 +77,10 @@ def update( """ cal_data = self.calibration_data self.last_calibration_datetime = calibration_datetime_str - + if len(self.calibration_parameters) == 0: + self.calibration_parameters = list(params.keys()) # Ensure all required calibration parameters were used - if not set(params.keys()) >= set(self.calibration_parameters): + elif not set(params.keys()) >= set(self.calibration_parameters): raise Exception( "Not enough parameters specified to update calibration.\n" + f"Required parameters are {self.calibration_parameters}" From bb2c35f6b99ae47f821589174298f7af13fb0826 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 10:12:26 -0700 Subject: [PATCH 13/16] update cal only with sensor parameters. --- scos_actions/actions/calibrate_y_factor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 11ced67c..0b1f42e2 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -253,7 +253,7 @@ def calibrate(self, params): assert ( sample_rate == noise_off_measurement_result["sample_rate"] ), "Sample rate mismatch" - + sigan_params = {k: v for k, v in params.items() if k in SIGAN_SETTINGS_KEYS} # Apply IIR filtering to both captures if configured if self.iir_apply: # Estimate of IIR filter ENBW does NOT account for passband ripple in sensor transfer function! @@ -264,7 +264,6 @@ def calibrate(self, params): else: logger.debug("Skipping IIR filtering") # Get ENBW from sensor calibration - sigan_params = {k: v for k, v in params.items() if k in SIGAN_SETTINGS_KEYS} assert set(sensor_calibration.calibration_parameters) <= set( sigan_params.keys() ), f"Action parameters do not include all required calibration parameters" @@ -289,7 +288,7 @@ def calibrate(self, params): # Update sensor calibration with results sensor_calibration.update( - params, + sigan_params, utils.get_datetime_str_now(), gain, noise_figure, From 15e7b2da10864d56694c440a5d4018d5d6a91f3b Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 12:12:21 -0700 Subject: [PATCH 14/16] Raise exception if attempting to calibrate without iir filter with default cal file. --- scos_actions/actions/calibrate_y_factor.py | 7 ++++++- scos_actions/calibration/__init__.py | 13 ++++++++++--- scos_actions/settings.py | 3 ++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 0b1f42e2..bc975e65 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -78,7 +78,7 @@ from scos_actions import utils from scos_actions.actions.interfaces.action import Action -from scos_actions.calibration import sensor_calibration +from scos_actions.calibration import sensor_calibration, default_sensor_calibration from scos_actions.hardware.mocks.mock_gps import MockGPS from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS from scos_actions.settings import SENSOR_CALIBRATION_FILE @@ -262,6 +262,11 @@ def calibrate(self, params): noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) else: + if default_sensor_calibration: + raise Exception( + "Calibrations without IIR filter cannot be performed with default calibration." + ) + logger.debug("Skipping IIR filtering") # Get ENBW from sensor calibration assert set(sensor_calibration.calibration_parameters) <= set( diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index 7d2db429..13b5568d 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -20,7 +20,6 @@ def get_sigan_calibration(sigan_cal_file: str) -> Calibration: :return: The signal analyzer ``Calibration`` object. """ try: - check_for_default_calibration(sigan_cal_file, "Sigan") sigan_cal = load_from_json(sigan_cal_file) except Exception: sigan_cal = None @@ -37,7 +36,6 @@ def get_sensor_calibration(sensor_cal_file: str) -> Calibration: :return: The sensor ``Calibration`` object. """ try: - check_for_default_calibration(sensor_cal_file, "Sensor") sensor_cal = load_from_json(sensor_cal_file) except Exception: sensor_cal = None @@ -45,11 +43,14 @@ def get_sensor_calibration(sensor_cal_file: str) -> Calibration: return sensor_cal -def check_for_default_calibration(cal_file_path: str, cal_type: str): +def check_for_default_calibration(cal_file_path: str, cal_type: str) -> bool: + default_cal = False if cal_file_path == DEFAULT_CALIBRATION_FILE: + default_cal = True logger.warning( f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" ) + return default_cal sensor_calibration = None @@ -62,6 +63,9 @@ def check_for_default_calibration(cal_file_path: str, cal_type: str): ) else: logger.debug(f"Loading sensor cal file: {SENSOR_CALIBRATION_FILE}") + default_sensor_calibration = check_for_default_calibration( + SENSOR_CALIBRATION_FILE, "Sensor" + ) sensor_calibration = get_sensor_calibration(SENSOR_CALIBRATION_FILE) sigan_calibration = None @@ -73,6 +77,9 @@ def check_for_default_calibration(cal_file_path: str, cal_type: str): ) else: logger.debug(f"Loading sigan cal file: {SIGAN_CALIBRATION_FILE}") + default_sigan_calibration = check_for_default_calibration( + SIGAN_CALIBRATION_FILE, "Sigan" + ) sigan_calibration = get_sigan_calibration(SIGAN_CALIBRATION_FILE) if sensor_calibration: diff --git a/scos_actions/settings.py b/scos_actions/settings.py index 7b628801..00cf702a 100644 --- a/scos_actions/settings.py +++ b/scos_actions/settings.py @@ -1,4 +1,5 @@ import logging +from os import path from pathlib import Path from django.conf import settings @@ -12,7 +13,7 @@ ACTION_DEFINITIONS_DIR = CONFIG_DIR / "actions" if not settings.configured or not hasattr(settings, "DEFAULT_CALIBRATION_FILE"): - DEFAULT_CALIBRATION_FILE = "" + DEFAULT_CALIBRATION_FILE = path.join(CONFIG_DIR, "default_calibration.json") else: DEFAULT_CALIBRATION_FILE = settings.DEFAULT_CALIBRATION_FILE From d47d76300894e2c97bb8e62e4ab458b1d4b52a16 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Wed, 3 Jan 2024 13:39:46 -0700 Subject: [PATCH 15/16] default default cal flags to False. --- scos_actions/calibration/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index 13b5568d..4014a2a0 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -69,6 +69,8 @@ def check_for_default_calibration(cal_file_path: str, cal_type: str) -> bool: sensor_calibration = get_sensor_calibration(SENSOR_CALIBRATION_FILE) sigan_calibration = None +default_sensor_calibration = False +default_sigan_calibration = False if SIGAN_CALIBRATION_FILE is None: logger.warning("Sigan calibration file is None. Not loading calibration file.") elif not path.exists(SIGAN_CALIBRATION_FILE): From fd5eaa5f43564cdbc11d5317bb6e71d22599b246 Mon Sep 17 00:00:00 2001 From: Doug Boulware Date: Fri, 5 Jan 2024 10:14:56 -0700 Subject: [PATCH 16/16] Fix checking if sensor and sigan cal files are set. --- scos_actions/calibration/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index 4014a2a0..e78f4c7e 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -54,8 +54,10 @@ def check_for_default_calibration(cal_file_path: str, cal_type: str) -> bool: sensor_calibration = None -if SENSOR_CALIBRATION_FILE is None: - logger.warning("Sensor calibration file is None. Not loading calibration file.") +if SENSOR_CALIBRATION_FILE is None or SENSOR_CALIBRATION_FILE == "": + logger.warning( + "No sensor calibration file specified. Not loading calibration file." + ) elif not path.exists(SENSOR_CALIBRATION_FILE): logger.warning( SENSOR_CALIBRATION_FILE @@ -71,8 +73,8 @@ def check_for_default_calibration(cal_file_path: str, cal_type: str) -> bool: sigan_calibration = None default_sensor_calibration = False default_sigan_calibration = False -if SIGAN_CALIBRATION_FILE is None: - logger.warning("Sigan calibration file is None. Not loading calibration file.") +if SIGAN_CALIBRATION_FILE is None or SIGAN_CALIBRATION_FILE == "": + logger.warning("No sigan calibration file specified. Not loading calibration file.") elif not path.exists(SIGAN_CALIBRATION_FILE): logger.warning( SIGAN_CALIBRATION_FILE + " does not exist. Not loading sigan calibration file."