From b200f553587550bf03bc81b5acc497a4294cc65e Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Fri, 19 Jan 2024 13:05:44 -0500
Subject: [PATCH 01/62] rename Calibration to SensorCalibration

---
 .../{calibration.py => sensor_calibration.py}      |  6 +++---
 scos_actions/hardware/mocks/mock_sigan.py          |  6 +++---
 scos_actions/hardware/sigan_iface.py               | 14 +++++++-------
 3 files changed, 13 insertions(+), 13 deletions(-)
 rename scos_actions/calibration/{calibration.py => sensor_calibration.py} (98%)

diff --git a/scos_actions/calibration/calibration.py b/scos_actions/calibration/sensor_calibration.py
similarity index 98%
rename from scos_actions/calibration/calibration.py
rename to scos_actions/calibration/sensor_calibration.py
index 2506d825..627202dc 100644
--- a/scos_actions/calibration/calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -10,7 +10,7 @@
 
 
 @dataclass
-class Calibration:
+class SensorCalibration:
     last_calibration_datetime: str
     calibration_parameters: List[str]
     calibration_data: dict
@@ -121,7 +121,7 @@ def update(
             outfile.write(json.dumps(cal_dict))
 
 
-def load_from_json(fname: Path, is_default: bool) -> Calibration:
+def load_from_json(fname: Path, is_default: bool) -> SensorCalibration:
     """
     Load a calibration from a JSON file.
 
@@ -155,7 +155,7 @@ def load_from_json(fname: Path, is_default: bool) -> Calibration:
             + f"Required fields: {required_keys}\n"
         )
     # Create and return the Calibration object
-    return Calibration(
+    return SensorCalibration(
         calibration["last_calibration_datetime"],
         calibration["calibration_parameters"],
         calibration["calibration_data"],
diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 1876851b..71cc71ba 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -4,7 +4,7 @@
 from typing import Optional
 
 import numpy as np
-from scos_actions.calibration.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import get_datetime_str_now
 
@@ -28,8 +28,8 @@ class MockSignalAnalyzer(SignalAnalyzerInterface):
 
     def __init__(
         self,
-        sensor_cal: Optional[Calibration] = None,
-        sigan_cal: Optional[Calibration] = None,
+        sensor_cal: Optional[SensorCalibration] = None,
+        sigan_cal: Optional[SensorCalibration] = None,
         randomize_values: bool = False,
     ):
         super().__init__(sensor_cal, sigan_cal)
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index 7ed01d45..f0eccf2c 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -4,7 +4,7 @@
 from typing import Dict, Optional
 
 from its_preselector.web_relay import WebRelay
-from scos_actions.calibration.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.utils import power_cycle_sigan
 from scos_actions.utils import convert_string_to_millisecond_iso_format
 
@@ -25,8 +25,8 @@
 class SignalAnalyzerInterface(ABC):
     def __init__(
         self,
-        sensor_cal: Optional[Calibration] = None,
-        sigan_cal: Optional[Calibration] = None,
+        sensor_cal: Optional[SensorCalibration] = None,
+        sigan_cal: Optional[SensorCalibration] = None,
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
         self.sensor_calibration_data = {}
@@ -166,17 +166,17 @@ def model(self, value: str):
         self._model = value
 
     @property
-    def sensor_calibration(self) -> Calibration:
+    def sensor_calibration(self) -> SensorCalibration:
         return self._sensor_calibration
 
     @sensor_calibration.setter
-    def sensor_calibration(self, cal: Calibration):
+    def sensor_calibration(self, cal: SensorCalibration):
         self._sensor_calibration = cal
 
     @property
-    def sigan_calibration(self) -> Calibration:
+    def sigan_calibration(self) -> SensorCalibration:
         return self._sigan_calibration
 
     @sigan_calibration.setter
-    def sigan_calibration(self, cal: Calibration):
+    def sigan_calibration(self, cal: SensorCalibration):
         self._sigan_calibration = cal

From 367e4db0e4a2975c1bf80b6d1415302840b7344f Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:08:31 -0500
Subject: [PATCH 02/62] refactor calibration to use a base class

---
 .../calibration/interfaces/__init__.py        |   0
 .../calibration/interfaces/calibration.py     | 119 ++++++++++++
 .../calibration/sensor_calibration.py         | 174 +++---------------
 .../calibration/tests/test_calibration.py     |  21 +--
 scos_actions/calibration/utils.py             |  57 ++++++
 scos_actions/signal_processing/calibration.py |  13 +-
 6 files changed, 215 insertions(+), 169 deletions(-)
 create mode 100644 scos_actions/calibration/interfaces/__init__.py
 create mode 100644 scos_actions/calibration/interfaces/calibration.py
 create mode 100644 scos_actions/calibration/utils.py

diff --git a/scos_actions/calibration/interfaces/__init__.py b/scos_actions/calibration/interfaces/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
new file mode 100644
index 00000000..9fe9aac3
--- /dev/null
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -0,0 +1,119 @@
+import dataclasses
+import json
+import logging
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, List
+
+from scos_actions.calibration.utils import filter_by_parameter
+
+logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class Calibration:
+    calibration_parameters: List[str]
+    calibration_data: dict
+    is_default: bool
+    file_path: Path
+
+    def __post_init__(self):
+        # Convert key names in data to strings
+        # This means that formatting will always match between
+        # native types provided in Python and data loaded from JSON
+        self.calibration_data = json.loads(json.dumps(self.calibration_data))
+
+    def get_calibration_dict(self, cal_params: List[Any]) -> dict:
+        """
+        Get calibration data closest to the specified parameter values.
+
+        :param cal_params: List of calibration parameter values. For example,
+            if ``calibration_parameters`` are ``["sample_rate", "gain"]``,
+            then the input to this method could be ``["15360000.0", "40"]``.
+        :return: The calibration data corresponding to the input parameter values.
+        """
+        cal_data = self.calibration_data
+        for i, setting_value in enumerate(cal_params):
+            setting = self.calibration_parameters[i]
+            logger.debug(f"Looking up calibration for {setting} at {setting_value}")
+            cal_data = filter_by_parameter(cal_data, setting_value)
+        logger.debug(f"Got calibration data: {cal_data}")
+
+        return cal_data
+
+    def _retrieve_data_to_update(self, params: dict) -> dict:
+        """
+        Locate the calibration data entry to update, based on a set
+        of calibration parameters.
+
+        :param params: Parameters used for calibration. This must include
+            entries for all of the ``Calibration.calibration_parameters``
+            Example: ``{"sample_rate": 14000000.0, "attenuation": 10.0}``
+        :return: A dict containing the existing calibration entry at
+            the specified parameter set, which may be empty if none exists.
+        """
+        # Use params keys as calibration_parameters if none exist
+        if len(self.calibration_parameters) == 0:
+            logger.warning(
+                f"Setting required calibration parameters to {list(params.keys())}"
+            )
+            self.calibration_parameters = list(params.keys())
+        elif not set(params.keys()) >= set(self.calibration_parameters):
+            # Otherwise ensure all required parameters were used
+            raise Exception(
+                "Not enough parameters specified to update calibration.\n"
+                + f"Required parameters are {self.calibration_parameters}"
+            )
+
+        # Retrieve the existing calibration data entry based on
+        # the provided parameters and their values
+        data_entry = self.calibration_data
+        for parameter in self.calibration_parameters:
+            value = str(params[parameter]).lower()
+            logger.debug(f"Updating calibration at {parameter} = {value}")
+            try:
+                data_entry = data_entry[value]
+            except KeyError:
+                logger.debug(
+                    f"Creating required calibration data field for {parameter} = {value}"
+                )
+                data_entry[value] = {}
+                data_entry = data_entry[value]
+        return data_entry
+
+    @abstractmethod
+    def update():
+        """Update the calibration data"""
+        pass
+
+    @classmethod
+    def from_json(cls, fname: Path, is_default: bool):
+        """
+        Load a calibration from a JSON file.
+
+        The JSON file must contain top-level fields:
+            ``calibration_parameters``
+            ``calibration_data``
+
+        :param fname: The ``Path`` to the JSON calibration file.
+        :param is_default: If True, the loaded calibration file
+            is treated as the default calibration file.
+        :raises Exception: If the provided file does not include
+            the required keys.
+        :return: The ``Calibration`` object generated from the file.
+        """
+        with open(fname) as file:
+            calibration = json.load(file)
+
+        # Check that the required fields are in the dict
+        required_keys = set(dataclasses.fields(cls).keys())
+
+        if not set(calibration.keys()) >= required_keys:
+            raise Exception(
+                "Loaded calibration dictionary is missing required fields."
+                + f"Existing fields: {set(calibration.keys())}\n"
+                + f"Required fields: {required_keys}\n"
+            )
+
+        # Create and return the Calibration object
+        return cls(is_default=is_default, file_path=fname, **calibration)
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 627202dc..3132c9d1 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -4,25 +4,15 @@
 from pathlib import Path
 from typing import Dict, List, Union
 
-from scos_actions.signal_processing.calibration import CalibrationException
+from scos_actions.calibration.interfaces.calibration import Calibration
 
 logger = logging.getLogger(__name__)
 
 
 @dataclass
-class SensorCalibration:
+class SensorCalibration(Calibration):
     last_calibration_datetime: str
-    calibration_parameters: List[str]
-    calibration_data: dict
     clock_rate_lookup_by_sample_rate: List[Dict[str, float]]
-    is_default: bool
-    file_path: Path
-
-    def __post_init__(self):
-        # Convert key names in calibration_data to strings
-        # This means that formatting will always match between
-        # native types provided in Python and data loaded from JSON
-        self.calibration_data = json.loads(json.dumps(self.calibration_data))
 
     def get_clock_rate(self, sample_rate: Union[float, int]) -> Union[float, int]:
         """Find the clock rate (Hz) using the given sample_rate (samples per second)"""
@@ -31,25 +21,6 @@ def get_clock_rate(self, sample_rate: Union[float, int]) -> Union[float, int]:
                 return mapping["clock_frequency"]
         return sample_rate
 
-    def get_calibration_dict(self, cal_params: List[Union[float, int, bool]]) -> dict:
-        """
-        Get calibration data closest to the specified parameter values.
-
-        :param cal_params: List of calibration parameter values. For example,
-            if ``calibration_parameters`` are ``["sample_rate", "gain"]``,
-            then the input to this method could be ``["15360000.0", "40"]``.
-        :return: The calibration data corresponding to the input parameter values.
-        """
-
-        cal_data = self.calibration_data
-        for i, setting_value in enumerate(cal_params):
-            setting = self.calibration_parameters[i]
-            logger.debug(f"Looking up calibration for {setting} at {setting_value}")
-            cal_data = filter_by_parameter(cal_data, setting_value)
-        logger.debug(f"Got calibration data: {cal_data}")
-
-        return cal_data
-
     def update(
         self,
         params: dict,
@@ -76,32 +47,14 @@ def update(
         :param file_path: File path for saving the updated calibration data.
         :raises Exception:
         """
-        cal_data = self.calibration_data
-        self.last_calibration_datetime = calibration_datetime_str
-        if len(self.calibration_parameters) == 0:
-            self.calibration_parameters = list(params.keys())
-        # Ensure all required calibration parameters were used
-        elif not set(params.keys()) >= set(self.calibration_parameters):
-            raise Exception(
-                "Not enough parameters specified to update calibration.\n"
-                + f"Required parameters are {self.calibration_parameters}"
-            )
+        # Get existing calibration data entry which will be updated
+        data_entry = self._retrieve_data_to_update(params)
 
-        # Get calibration entry by parameters used
-        for parameter in self.calibration_parameters:
-            value = str(params[parameter]).lower()
-            logger.debug(f"Updating calibration at {parameter} = {value}")
-            try:
-                cal_data = cal_data[value]
-            except KeyError:
-                logger.debug(
-                    f"Creating required calibration data field for {parameter} = {value}"
-                )
-                cal_data[value] = {}
-                cal_data = cal_data[value]
+        # Update last calibration datetime
+        self.last_calibration_datetime = calibration_datetime_str
 
-        # Update calibration data
-        cal_data.update(
+        # Update calibration data entry (updates entry in self.calibration_data)
+        data_entry.update(
             {
                 "datetime": calibration_datetime_str,
                 "gain": gain_dB,
@@ -120,95 +73,22 @@ def update(
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(cal_dict))
 
-
-def load_from_json(fname: Path, is_default: bool) -> SensorCalibration:
-    """
-    Load a calibration from a JSON file.
-
-    The JSON file must contain top-level fields:
-        ``last_calibration_datetime``
-        ``calibration_parameters``
-        ``calibration_data``
-        ``clock_rate_lookup_by_sample_rate``
-
-    :param fname: The ``Path`` to the JSON calibration file.
-    :param is_default: If True, the loaded calibration file
-        is treated as the default calibration file.
-    :raises Exception: If the provided file does not include
-        the required keys.
-    :return: The ``Calibration`` object generated from the file.
-    """
-    with open(fname) as file:
-        calibration = json.load(file)
-    # Check that the required fields are in the dict
-    required_keys = {
-        "last_calibration_datetime",
-        "calibration_data",
-        "clock_rate_lookup_by_sample_rate",
-        "calibration_parameters",
-    }
-
-    if not calibration.keys() >= required_keys:
-        raise Exception(
-            "Loaded calibration dictionary is missing required fields."
-            + f"Existing fields: {set(calibration.keys())}\n"
-            + f"Required fields: {required_keys}\n"
-        )
-    # Create and return the Calibration object
-    return SensorCalibration(
-        calibration["last_calibration_datetime"],
-        calibration["calibration_parameters"],
-        calibration["calibration_data"],
-        calibration["clock_rate_lookup_by_sample_rate"],
-        is_default,
-        fname,
-    )
-
-
-def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> dict:
-    """
-    Select a certain element by the value of a top-level key in a dictionary.
-
-    This method should be recursively called to select calibration
-    data matching a set of calibration parameters. The ordering of
-    nested dictionaries should match the ordering of the required
-    calibration parameters in the calibration file.
-
-    If ``value`` is a float or bool, ``str(value).lower()`` is used
-    as the dictionary key. If ``value`` is an int, and the previous
-    approach does not work, ``str(float(value))`` is attempted. This
-    allows for value ``1`` to match a key ``"1.0"``.
-
-    :param calibrations: Calibration data dictionary.
-    :param value: The parameter value for filtering. This value should
-        exist as a top-level key in ``calibrations``.
-    :raises CalibrationException: If ``value`` cannot be matched to a
-        top-level key in ``calibrations``, or if ``calibrations`` is not
-        a dict.
-    :return: The value of ``calibrations[value]``, which should be a dict.
-    """
-    try:
-        filtered_data = calibrations.get(str(value).lower(), None)
-        if filtered_data is None and isinstance(value, int):
-            # Try equivalent float for ints, i.e., match "1.0" to 1
-            filtered_data = calibrations.get(str(float(value)), None)
-        if filtered_data is None and isinstance(value, float) and value.is_integer():
-            # Check for, e.g., key '25' if value is '25.0'
-            filtered_data = calibrations.get(str(int(value)), None)
-        if filtered_data is None:
-            raise KeyError
-        else:
-            return filtered_data
-    except AttributeError as e:
-        # calibrations does not have ".get()"
-        # Generally means that calibrations is None or not a dict
-        msg = f"Provided calibration data is not a dict: {calibrations}"
-        raise CalibrationException(msg)
-    except KeyError as e:
-        msg = (
-            f"Could not locate calibration data at {value}"
-            + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
-            + f"\nUsing calibration data: {calibrations}"
-        )
-        raise CalibrationException(msg)
+    # @classmethod
+    # def from_json(cls, fname: Path, is_default: bool):
+    #     """
+    #     Load a sensor calibration from a JSON file.
+
+    #     The JSON file must contain top-level fields:
+    #         ``calibration_parameters``
+    #         ``calibration_data``
+    #         ``last_calibration_datetime``
+    #         ``clock_rate_lookup_by_sample_rate``
+
+    #     :param fname: The ``Path`` to the JSON calibration file.
+    #     :param is_default: If True, the loaded calibration file
+    #         is treated as the default calibration file.
+    #     :raises Exception: If the provided file does not include
+    #         the required keys.
+    #     :return: The ``Calibration`` object generated from the file.
+    #     """
+    #     return super().from_json(fname, is_default)
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 9e503acd..c4484a90 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -8,18 +8,13 @@
 from pathlib import Path
 
 import pytest
-
-from scos_actions.calibration.calibration import (
-    Calibration,
-    filter_by_parameter,
-    load_from_json,
-)
-from scos_actions.signal_processing.calibration import CalibrationException
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
 from scos_actions.tests.resources.utils import easy_gain
 from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
 
 
-class TestCalibrationFile:
+class TestSensorCalibrationFile:
     # Ensure we load the test file
     setup_complete = False
 
@@ -180,7 +175,7 @@ def setup_calibration_file(self, tmpdir):
             json.dump(cal_data, file, indent=4)
 
         # Load the data back in
-        self.sample_cal = load_from_json(self.calibration_file, False)
+        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
 
         # Create a list of previous points to ensure that we don't repeat
         self.pytest_points = []
@@ -229,7 +224,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
             200.0: {100.0: {"NF": "NF at 200, 100", "Gain": "Gain at 200, 100"}},
         }
         clock_rate_lookup_by_sample_rate = {}
-        cal = Calibration(
+        cal = SensorCalibration(
             calibration_datetime,
             calibration_params,
             calibration_data,
@@ -249,7 +244,7 @@ def test_get_calibration_dict_within_range(self):
         }
         clock_rate_lookup_by_sample_rate = {}
         test_cal_path = Path("test_calibration.json")
-        cal = Calibration(
+        cal = SensorCalibration(
             calibration_datetime,
             calibration_params,
             calibration_data,
@@ -291,7 +286,7 @@ def test_update(self):
         calibration_data = {100.0: {200.0: {"noise_figure": 0, "gain": 0}}}
         clock_rate_lookup_by_sample_rate = {}
         test_cal_path = Path("test_calibration.json")
-        cal = Calibration(
+        cal = SensorCalibration(
             calibration_datetime,
             calibration_params,
             calibration_data,
@@ -302,7 +297,7 @@ def test_update(self):
         action_params = {"sample_rate": 100.0, "frequency": 200.0}
         update_time = get_datetime_str_now()
         cal.update(action_params, update_time, 30.0, 5.0, 21)
-        cal_from_file = load_from_json(test_cal_path, False)
+        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
         test_cal_path.unlink()
         file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
         cal_time_utc = parse_datetime_iso_format_str(update_time)
diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
new file mode 100644
index 00000000..b3aebdeb
--- /dev/null
+++ b/scos_actions/calibration/utils.py
@@ -0,0 +1,57 @@
+from typing import Union
+
+
+class CalibrationException(Exception):
+    """Basic exception handling for calibration functions."""
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+
+def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> dict:
+    """
+    Select a certain element by the value of a top-level key in a dictionary.
+
+    This method should be recursively called to select calibration
+    data matching a set of calibration parameters. The ordering of
+    nested dictionaries should match the ordering of the required
+    calibration parameters in the calibration file.
+
+    If ``value`` is a float or bool, ``str(value).lower()`` is used
+    as the dictionary key. If ``value`` is an int, and the previous
+    approach does not work, ``str(float(value))`` is attempted. This
+    allows for value ``1`` to match a key ``"1.0"``.
+
+    :param calibrations: Calibration data dictionary.
+    :param value: The parameter value for filtering. This value should
+        exist as a top-level key in ``calibrations``.
+    :raises CalibrationException: If ``value`` cannot be matched to a
+        top-level key in ``calibrations``, or if ``calibrations`` is not
+        a dict.
+    :return: The value of ``calibrations[value]``, which should be a dict.
+    """
+    try:
+        filtered_data = calibrations.get(str(value).lower(), None)
+        if filtered_data is None and isinstance(value, int):
+            # Try equivalent float for ints, i.e., match "1.0" to 1
+            filtered_data = calibrations.get(str(float(value)), None)
+        if filtered_data is None and isinstance(value, float) and value.is_integer():
+            # Check for, e.g., key '25' if value is '25.0'
+            filtered_data = calibrations.get(str(int(value)), None)
+        if filtered_data is None:
+            raise KeyError
+        else:
+            return filtered_data
+    except AttributeError as e:
+        # calibrations does not have ".get()"
+        # Generally means that calibrations is None or not a dict
+        msg = f"Provided calibration data is not a dict: {calibrations}"
+        raise CalibrationException(msg)
+    except KeyError as e:
+        msg = (
+            f"Could not locate calibration data at {value}"
+            + f"\nAttempted lookup using key '{str(value).lower()}'"
+            + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
+            + f"\nUsing calibration data: {calibrations}"
+        )
+        raise CalibrationException(msg)
diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py
index 41ef5d71..757dfc50 100644
--- a/scos_actions/signal_processing/calibration.py
+++ b/scos_actions/signal_processing/calibration.py
@@ -3,7 +3,9 @@
 
 import numpy as np
 from its_preselector.preselector import Preselector
+from numpy.typing import NDArray
 from scipy.constants import Boltzmann
+from scos_actions.calibration.utils import CalibrationException
 from scos_actions.signal_processing.unit_conversion import (
     convert_celsius_to_fahrenheit,
     convert_celsius_to_kelvins,
@@ -15,16 +17,9 @@
 logger = logging.getLogger(__name__)
 
 
-class CalibrationException(Exception):
-    """Basic exception handling for calibration functions."""
-
-    def __init__(self, msg):
-        super().__init__(msg)
-
-
 def y_factor(
-    pwr_noise_on_watts: np.ndarray,
-    pwr_noise_off_watts: np.ndarray,
+    pwr_noise_on_watts: NDArray,
+    pwr_noise_off_watts: NDArray,
     enr_linear: float,
     enbw_hz: float,
     temp_kelvins: float = 300.0,

From 9f68928781f94c94009fda858d7da4cf0c14111d Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:42:35 -0500
Subject: [PATCH 03/62] fix tests, add sensor_uid to SensorCalibration

---
 .../calibration/interfaces/calibration.py     |  33 +-
 .../calibration/sensor_calibration.py         |  22 +-
 .../calibration/tests/test_calibration.py     | 318 +-----------------
 .../tests/test_sensor_calibration.py          | 315 +++++++++++++++++
 4 files changed, 340 insertions(+), 348 deletions(-)
 create mode 100644 scos_actions/calibration/tests/test_sensor_calibration.py

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 9fe9aac3..debe42ba 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -84,16 +84,16 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
     @abstractmethod
     def update():
         """Update the calibration data"""
-        pass
+        raise NotImplementedError
 
     @classmethod
     def from_json(cls, fname: Path, is_default: bool):
         """
         Load a calibration from a JSON file.
 
-        The JSON file must contain top-level fields:
-            ``calibration_parameters``
-            ``calibration_data``
+        The JSON file must contain top-level fields
+        with names identical to the dataclass fields for
+        the class being constructed.
 
         :param fname: The ``Path`` to the JSON calibration file.
         :param is_default: If True, the loaded calibration file
@@ -104,15 +104,26 @@ def from_json(cls, fname: Path, is_default: bool):
         """
         with open(fname) as file:
             calibration = json.load(file)
-
-        # Check that the required fields are in the dict
-        required_keys = set(dataclasses.fields(cls).keys())
-
-        if not set(calibration.keys()) >= required_keys:
+        cal_file_keys = set(calibration.keys())
+
+        # Check that only the required fields are in the dict
+        required_keys = {f.name for f in dataclasses.fields(cls)}
+        required_keys -= {"is_default", "file_path"}  # are not required in JSON
+        if cal_file_keys == required_keys:
+            pass
+        elif cal_file_keys >= required_keys:
+            extra_keys = cal_file_keys - required_keys
+            logger.warning(
+                f"Loaded calibration file contains fields which will be ignored: {extra_keys}"
+            )
+            for k in extra_keys:
+                calibration.pop(k, None)
+        else:
             raise Exception(
-                "Loaded calibration dictionary is missing required fields."
-                + f"Existing fields: {set(calibration.keys())}\n"
+                "Loaded calibration dictionary is missing required fields.\n"
+                + f"Existing fields: {cal_file_keys}\n"
                 + f"Required fields: {required_keys}\n"
+                + f"Missing fields: {required_keys - cal_file_keys}"
             )
 
         # Create and return the Calibration object
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 3132c9d1..11865206 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -13,6 +13,7 @@
 class SensorCalibration(Calibration):
     last_calibration_datetime: str
     clock_rate_lookup_by_sample_rate: List[Dict[str, float]]
+    sensor_uid: str
 
     def get_clock_rate(self, sample_rate: Union[float, int]) -> Union[float, int]:
         """Find the clock rate (Hz) using the given sample_rate (samples per second)"""
@@ -66,29 +67,10 @@ def update(
         # Write updated calibration data to file
         cal_dict = {
             "last_calibration_datetime": self.last_calibration_datetime,
+            "sensor_uid": self.sensor_uid,
             "calibration_parameters": self.calibration_parameters,
             "clock_rate_lookup_by_sample_rate": self.clock_rate_lookup_by_sample_rate,
             "calibration_data": self.calibration_data,
         }
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(cal_dict))
-
-    # @classmethod
-    # def from_json(cls, fname: Path, is_default: bool):
-    #     """
-    #     Load a sensor calibration from a JSON file.
-
-    #     The JSON file must contain top-level fields:
-    #         ``calibration_parameters``
-    #         ``calibration_data``
-    #         ``last_calibration_datetime``
-    #         ``clock_rate_lookup_by_sample_rate``
-
-    #     :param fname: The ``Path`` to the JSON calibration file.
-    #     :param is_default: If True, the loaded calibration file
-    #         is treated as the default calibration file.
-    #     :raises Exception: If the provided file does not include
-    #         the required keys.
-    #     :return: The ``Calibration`` object generated from the file.
-    #     """
-    #     return super().from_json(fname, is_default)
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index c4484a90..3d743de1 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -1,317 +1 @@
-"""Test aspects of ScaleFactors."""
-
-import datetime
-import json
-import random
-from copy import deepcopy
-from math import isclose
-from pathlib import Path
-
-import pytest
-from scos_actions.calibration.sensor_calibration import SensorCalibration
-from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
-from scos_actions.tests.resources.utils import easy_gain
-from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
-
-
-class TestSensorCalibrationFile:
-    # Ensure we load the test file
-    setup_complete = False
-
-    def rand_index(self, l):
-        """Get a random index for a list"""
-        return random.randint(0, len(l) - 1)
-
-    def check_duplicate(self, sr, f, g):
-        """Check if a set of points was already tested"""
-        for pt in self.pytest_points:
-            duplicate_f = f == pt["frequency"]
-            duplicate_g = g == pt["setting_value"]
-            duplicate_sr = sr == pt["sample_rate"]
-            if duplicate_f and duplicate_g and duplicate_sr:
-                return True
-
-    def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
-        """Test the calculated value against the algorithm
-        Parameters:
-            sr, f, g -> Set values for the mock USRP
-            reason: Test case string for failure reference
-            sr_m, f_m, g_m -> Set values to use when calculating the expected value
-                              May differ in from actual set points in edge cases
-                              such as tuning in divisions or uncalibrated sample rate"""
-        # Check that the setup was completed
-        assert self.setup_complete, "Setup was not completed"
-
-        # If this point was tested before, skip it (triggering a new one)
-        if self.check_duplicate(sr, f, g):
-            return False
-
-        # If the point doesn't have modified inputs, use the algorithm ones
-        if not f_m:
-            f_m = f
-        if not g_m:
-            g_m = g
-        if not sr_m:
-            sr_m = sr
-
-        # Calculate what the scale factor should be
-        calc_gain_sigan = easy_gain(sr_m, f_m, g_m)
-
-        # Get the scale factor from the algorithm
-        interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
-        interp_gain_siggan = interp_cal_data["gain"]
-
-        # Save the point so we don't duplicate
-        self.pytest_points.append(
-            {
-                "sample_rate": int(sr),
-                "frequency": f,
-                "setting_value": g,
-                "gain": calc_gain_sigan,
-                "test": reason,
-            }
-        )
-
-        # Check if the point was calculated correctly
-        tolerance = 1e-5
-        msg = "Scale factor not correctly calculated!\r\n"
-        msg = f"{msg}    Expected value:   {calc_gain_sigan}\r\n"
-        msg = f"{msg}    Calculated value: {interp_gain_siggan}\r\n"
-        msg = f"{msg}    Tolerance: {tolerance}\r\n"
-        msg = f"{msg}    Test: {reason}\r\n"
-        msg = f"{msg}    Sample Rate: {sr / 1e6}({sr_m / 1e6})\r\n"
-        msg = f"{msg}    Frequency: {f / 1e6}({f_m / 1e6})\r\n"
-        msg = f"{msg}    Gain: {g}({g_m})\r\n"
-        msg = (
-            "{}    Formula: -1 * (Gain - Frequency[GHz] - Sample Rate[MHz])\r\n".format(
-                msg
-            )
-        )
-        if not isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance):
-            interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
-
-        assert isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance), msg
-        return True
-
-    @pytest.fixture(autouse=True)
-    def setup_calibration_file(self, tmpdir):
-        """Create the dummy calibration file in the pytest temp directory"""
-
-        # Only setup once
-        if self.setup_complete:
-            return
-
-        # Create and save the temp directory and file
-        self.tmpdir = tmpdir.strpath
-        self.calibration_file = "{}".format(tmpdir.join("dummy_cal_file.json"))
-
-        # Setup variables
-        self.dummy_noise_figure = 10
-        self.dummy_compression = -20
-        self.test_repeat_times = 3
-
-        # Sweep variables
-        self.sample_rates = [10e6, 15.36e6, 40e6]
-        self.gain_min = 40
-        self.gain_max = 60
-        self.gain_step = 10
-        gains = list(range(self.gain_min, self.gain_max, self.gain_step)) + [
-            self.gain_max
-        ]
-        self.frequency_min = 1000000000
-        self.frequency_max = 3400000000
-        self.frequency_step = 200000000
-        frequencies = list(
-            range(self.frequency_min, self.frequency_max, self.frequency_step)
-        ) + [self.frequency_max]
-        frequencies = sorted(frequencies)
-
-        # Start with blank cal data dicts
-        cal_data = {}
-
-        # Add the simple stuff to new cal format
-        cal_data["last_calibration_datetime"] = get_datetime_str_now()
-        cal_data["sensor_uid"] = "SAMPLE_CALIBRATION"
-
-        # Add SR/CF lookup table
-        cal_data["clock_rate_lookup_by_sample_rate"] = []
-        for sr in self.sample_rates:
-            cr = sr
-            while cr <= 40e6:
-                cr *= 2
-            cr /= 2
-            cal_data["clock_rate_lookup_by_sample_rate"].append(
-                {"sample_rate": int(sr), "clock_frequency": int(cr)}
-            )
-
-        # Create the JSON architecture for the calibration data
-        cal_data["calibration_data"] = {}
-        cal_data["calibration_parameters"] = ["sample_rate", "frequency", "gain"]
-        for k in range(len(self.sample_rates)):
-            cal_data_f = {}
-            for i in range(len(frequencies)):
-                cal_data_g = {}
-                for j in range(len(gains)):
-                    # Create the scale factor that ensures easy interpolation
-                    gain_sigan = easy_gain(
-                        self.sample_rates[k], frequencies[i], gains[j]
-                    )
-
-                    # Create the data point
-                    cal_data_point = {
-                        "gain": gain_sigan,
-                        "noise_figure": self.dummy_noise_figure,
-                        "1dB_compression_point": self.dummy_compression,
-                    }
-
-                    # Add the generated dicts to the parent lists
-                    cal_data_g[gains[j]] = deepcopy(cal_data_point)
-                cal_data_f[frequencies[i]] = deepcopy(cal_data_g)
-
-            cal_data["calibration_data"][self.sample_rates[k]] = deepcopy(cal_data_f)
-
-        # Write the new json file
-        with open(self.calibration_file, "w+") as file:
-            json.dump(cal_data, file, indent=4)
-
-        # Load the data back in
-        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
-
-        # Create a list of previous points to ensure that we don't repeat
-        self.pytest_points = []
-
-        # Create sweep lists for test points
-        self.srs = self.sample_rates
-        self.gi_s = list(range(self.gain_min, self.gain_max, self.gain_step))
-        self.fi_s = list(
-            range(self.frequency_min, self.frequency_max, self.frequency_step)
-        )
-        self.g_s = self.gi_s + [self.gain_max]
-        self.f_s = self.fi_s + [self.frequency_max]
-
-        # Don't repeat test setup
-        self.setup_complete = True
-
-    def test_filter_by_parameter_out_of_range(self):
-        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 400.0)
-            assert (
-                e_info.value.args[0]
-                == f"Could not locate calibration data at 400.0"
-                + f"\nAttempted lookup using key '400.0'"
-                + f"\nUsing calibration data: {calibrations}"
-            )
-
-    def test_filter_by_parameter_in_range_requires_match(self):
-        calibrations = {
-            200.0: {"Gain": "Gain at 200.0"},
-            300.0: {"Gain": "Gain at 300.0"},
-        }
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 150.0)
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 150.0"
-                + f"\nAttempted lookup using key '150.0'"
-                + f"\nUsing calibration data: {calibrations}"
-            )
-
-    def test_get_calibration_dict_exact_match_lookup(self):
-        calibration_datetime = datetime.datetime.now()
-        calibration_params = ["sample_rate", "frequency"]
-        calibration_data = {
-            100.0: {200.0: {"NF": "NF at 100, 200", "Gain": "Gain at 100, 200"}},
-            200.0: {100.0: {"NF": "NF at 200, 100", "Gain": "Gain at 200, 100"}},
-        }
-        clock_rate_lookup_by_sample_rate = {}
-        cal = SensorCalibration(
-            calibration_datetime,
-            calibration_params,
-            calibration_data,
-            clock_rate_lookup_by_sample_rate,
-            False,
-            Path(""),
-        )
-        cal_data = cal.get_calibration_dict([100.0, 200.0])
-        assert cal_data["NF"] == "NF at 100, 200"
-
-    def test_get_calibration_dict_within_range(self):
-        calibration_datetime = datetime.datetime.now()
-        calibration_params = calibration_params = ["sample_rate", "frequency"]
-        calibration_data = {
-            100.0: {200: {"NF": "NF at 100, 200"}, 300.0: "Cal data at 100,300"},
-            200.0: {100.0: {"NF": "NF at 200, 100"}},
-        }
-        clock_rate_lookup_by_sample_rate = {}
-        test_cal_path = Path("test_calibration.json")
-        cal = SensorCalibration(
-            calibration_datetime,
-            calibration_params,
-            calibration_data,
-            clock_rate_lookup_by_sample_rate,
-            False,
-            test_cal_path,
-        )
-        with pytest.raises(CalibrationException) as e_info:
-            cal_data = cal.get_calibration_dict([100.0, 250.0])
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 250.0"
-                + f"\nAttempted lookup using key '250.0'"
-                + f"\nUsing calibration data: {cal.calibration_data}"
-            )
-
-    def test_sf_bound_points(self):
-        """Test SF determination at boundary points"""
-        self.run_pytest_point(
-            self.srs[0], self.frequency_min, self.gain_min, "Testing boundary points"
-        )
-        self.run_pytest_point(
-            self.srs[0], self.frequency_max, self.gain_max, "Testing boundary points"
-        )
-
-    def test_sf_no_interpolation_points(self):
-        """Test points without interpolation"""
-        for i in range(4 * self.test_repeat_times):
-            while True:
-                g = self.g_s[self.rand_index(self.g_s)]
-                f = self.f_s[self.rand_index(self.f_s)]
-                if self.run_pytest_point(
-                    self.srs[0], f, g, "Testing no interpolation points"
-                ):
-                    break
-
-    def test_update(self):
-        calibration_datetime = get_datetime_str_now()
-        calibration_params = ["sample_rate", "frequency"]
-        calibration_data = {100.0: {200.0: {"noise_figure": 0, "gain": 0}}}
-        clock_rate_lookup_by_sample_rate = {}
-        test_cal_path = Path("test_calibration.json")
-        cal = SensorCalibration(
-            calibration_datetime,
-            calibration_params,
-            calibration_data,
-            clock_rate_lookup_by_sample_rate,
-            False,
-            test_cal_path,
-        )
-        action_params = {"sample_rate": 100.0, "frequency": 200.0}
-        update_time = get_datetime_str_now()
-        cal.update(action_params, update_time, 30.0, 5.0, 21)
-        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
-        test_cal_path.unlink()
-        file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
-        cal_time_utc = parse_datetime_iso_format_str(update_time)
-        assert file_utc_time.year == cal_time_utc.year
-        assert file_utc_time.month == cal_time_utc.month
-        assert file_utc_time.day == cal_time_utc.day
-        assert file_utc_time.hour == cal_time_utc.hour
-        assert file_utc_time.minute == cal_time_utc.minute
-        assert cal.calibration_data["100.0"]["200.0"]["gain"] == 30.0
-        assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
-        assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
-        assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
-
-    def test_filter_by_paramter_integer(self):
-        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
-        filtered_data = filter_by_parameter(calibrations, 200)
-        assert filtered_data is calibrations["200.0"]
+"""Test the Calibration base class."""
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
new file mode 100644
index 00000000..cf4403fa
--- /dev/null
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -0,0 +1,315 @@
+"""Test the SensorCalibration class."""
+
+import json
+import random
+from copy import deepcopy
+from math import isclose
+from pathlib import Path
+
+import pytest
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
+from scos_actions.tests.resources.utils import easy_gain
+from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
+
+
+class TestSensorCalibrationFile:
+    # Ensure we load the test file
+    setup_complete = False
+
+    def rand_index(self, l):
+        """Get a random index for a list"""
+        return random.randint(0, len(l) - 1)
+
+    def check_duplicate(self, sr, f, g):
+        """Check if a set of points was already tested"""
+        for pt in self.pytest_points:
+            duplicate_f = f == pt["frequency"]
+            duplicate_g = g == pt["setting_value"]
+            duplicate_sr = sr == pt["sample_rate"]
+            if duplicate_f and duplicate_g and duplicate_sr:
+                return True
+
+    def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
+        """Test the calculated value against the algorithm
+        Parameters:
+            sr, f, g -> Set values for the mock USRP
+            reason: Test case string for failure reference
+            sr_m, f_m, g_m -> Set values to use when calculating the expected value
+                              May differ in from actual set points in edge cases
+                              such as tuning in divisions or uncalibrated sample rate"""
+        # Check that the setup was completed
+        assert self.setup_complete, "Setup was not completed"
+
+        # If this point was tested before, skip it (triggering a new one)
+        if self.check_duplicate(sr, f, g):
+            return False
+
+        # If the point doesn't have modified inputs, use the algorithm ones
+        if not f_m:
+            f_m = f
+        if not g_m:
+            g_m = g
+        if not sr_m:
+            sr_m = sr
+
+        # Calculate what the scale factor should be
+        calc_gain_sigan = easy_gain(sr_m, f_m, g_m)
+
+        # Get the scale factor from the algorithm
+        interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+        interp_gain_siggan = interp_cal_data["gain"]
+
+        # Save the point so we don't duplicate
+        self.pytest_points.append(
+            {
+                "sample_rate": int(sr),
+                "frequency": f,
+                "setting_value": g,
+                "gain": calc_gain_sigan,
+                "test": reason,
+            }
+        )
+
+        # Check if the point was calculated correctly
+        tolerance = 1e-5
+        msg = "Scale factor not correctly calculated!\r\n"
+        msg = f"{msg}    Expected value:   {calc_gain_sigan}\r\n"
+        msg = f"{msg}    Calculated value: {interp_gain_siggan}\r\n"
+        msg = f"{msg}    Tolerance: {tolerance}\r\n"
+        msg = f"{msg}    Test: {reason}\r\n"
+        msg = f"{msg}    Sample Rate: {sr / 1e6}({sr_m / 1e6})\r\n"
+        msg = f"{msg}    Frequency: {f / 1e6}({f_m / 1e6})\r\n"
+        msg = f"{msg}    Gain: {g}({g_m})\r\n"
+        msg = (
+            "{}    Formula: -1 * (Gain - Frequency[GHz] - Sample Rate[MHz])\r\n".format(
+                msg
+            )
+        )
+        if not isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance):
+            interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+
+        assert isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance), msg
+        return True
+
+    @pytest.fixture(autouse=True)
+    def setup_calibration_file(self, tmpdir):
+        """Create the dummy calibration file in the pytest temp directory"""
+
+        # Only setup once
+        if self.setup_complete:
+            return
+
+        # Create and save the temp directory and file
+        self.tmpdir = tmpdir.strpath
+        self.calibration_file = "{}".format(tmpdir.join("dummy_cal_file.json"))
+
+        # Setup variables
+        self.dummy_noise_figure = 10
+        self.dummy_compression = -20
+        self.test_repeat_times = 3
+
+        # Sweep variables
+        self.sample_rates = [10e6, 15.36e6, 40e6]
+        self.gain_min = 40
+        self.gain_max = 60
+        self.gain_step = 10
+        gains = list(range(self.gain_min, self.gain_max, self.gain_step)) + [
+            self.gain_max
+        ]
+        self.frequency_min = 1000000000
+        self.frequency_max = 3400000000
+        self.frequency_step = 200000000
+        frequencies = list(
+            range(self.frequency_min, self.frequency_max, self.frequency_step)
+        ) + [self.frequency_max]
+        frequencies = sorted(frequencies)
+
+        # Start with blank cal data dicts
+        cal_data = {}
+
+        # Add the simple stuff to new cal format
+        cal_data["last_calibration_datetime"] = get_datetime_str_now()
+        cal_data["sensor_uid"] = "SAMPLE_CALIBRATION"
+
+        # Add SR/CF lookup table
+        cal_data["clock_rate_lookup_by_sample_rate"] = []
+        for sr in self.sample_rates:
+            cr = sr
+            while cr <= 40e6:
+                cr *= 2
+            cr /= 2
+            cal_data["clock_rate_lookup_by_sample_rate"].append(
+                {"sample_rate": int(sr), "clock_frequency": int(cr)}
+            )
+
+        # Create the JSON architecture for the calibration data
+        cal_data["calibration_data"] = {}
+        cal_data["calibration_parameters"] = ["sample_rate", "frequency", "gain"]
+        for k in range(len(self.sample_rates)):
+            cal_data_f = {}
+            for i in range(len(frequencies)):
+                cal_data_g = {}
+                for j in range(len(gains)):
+                    # Create the scale factor that ensures easy interpolation
+                    gain_sigan = easy_gain(
+                        self.sample_rates[k], frequencies[i], gains[j]
+                    )
+
+                    # Create the data point
+                    cal_data_point = {
+                        "gain": gain_sigan,
+                        "noise_figure": self.dummy_noise_figure,
+                        "1dB_compression_point": self.dummy_compression,
+                    }
+
+                    # Add the generated dicts to the parent lists
+                    cal_data_g[gains[j]] = deepcopy(cal_data_point)
+                cal_data_f[frequencies[i]] = deepcopy(cal_data_g)
+
+            cal_data["calibration_data"][self.sample_rates[k]] = deepcopy(cal_data_f)
+
+        # Write the new json file
+        with open(self.calibration_file, "w+") as file:
+            json.dump(cal_data, file, indent=4)
+
+        # Load the data back in
+        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
+
+        # Create a list of previous points to ensure that we don't repeat
+        self.pytest_points = []
+
+        # Create sweep lists for test points
+        self.srs = self.sample_rates
+        self.gi_s = list(range(self.gain_min, self.gain_max, self.gain_step))
+        self.fi_s = list(
+            range(self.frequency_min, self.frequency_max, self.frequency_step)
+        )
+        self.g_s = self.gi_s + [self.gain_max]
+        self.f_s = self.fi_s + [self.frequency_max]
+
+        # Don't repeat test setup
+        self.setup_complete = True
+
+    def test_filter_by_parameter_out_of_range(self):
+        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
+        with pytest.raises(CalibrationException) as e_info:
+            cal = filter_by_parameter(calibrations, 400.0)
+            assert (
+                e_info.value.args[0]
+                == f"Could not locate calibration data at 400.0"
+                + f"\nAttempted lookup using key '400.0'"
+                + f"\nUsing calibration data: {calibrations}"
+            )
+
+    def test_filter_by_parameter_in_range_requires_match(self):
+        calibrations = {
+            200.0: {"Gain": "Gain at 200.0"},
+            300.0: {"Gain": "Gain at 300.0"},
+        }
+        with pytest.raises(CalibrationException) as e_info:
+            cal = filter_by_parameter(calibrations, 150.0)
+            assert e_info.value.args[0] == (
+                f"Could not locate calibration data at 150.0"
+                + f"\nAttempted lookup using key '150.0'"
+                + f"\nUsing calibration data: {calibrations}"
+            )
+
+    def test_get_calibration_dict_exact_match_lookup(self):
+        calibration_datetime = get_datetime_str_now()
+        calibration_params = ["sample_rate", "frequency"]
+        calibration_data = {
+            100.0: {200.0: {"NF": "NF at 100, 200", "Gain": "Gain at 100, 200"}},
+            200.0: {100.0: {"NF": "NF at 200, 100", "Gain": "Gain at 200, 100"}},
+        }
+        cal = SensorCalibration(
+            calibration_parameters=calibration_params,
+            calibration_data=calibration_data,
+            is_default=False,
+            file_path=Path(""),
+            last_calibration_datetime=calibration_datetime,
+            clock_rate_lookup_by_sample_rate={},
+            sensor_uid="TESTING",
+        )
+        cal_data = cal.get_calibration_dict([100.0, 200.0])
+        assert cal_data["NF"] == "NF at 100, 200"
+
+    def test_get_calibration_dict_within_range(self):
+        calibration_datetime = get_datetime_str_now()
+        calibration_params = calibration_params = ["sample_rate", "frequency"]
+        calibration_data = {
+            100.0: {200: {"NF": "NF at 100, 200"}, 300.0: "Cal data at 100,300"},
+            200.0: {100.0: {"NF": "NF at 200, 100"}},
+        }
+        cal = SensorCalibration(
+            calibration_parameters=calibration_params,
+            calibration_data=calibration_data,
+            is_default=False,
+            file_path=Path("test_calibration.json"),
+            last_calibration_datetime=calibration_datetime,
+            clock_rate_lookup_by_sample_rate={},
+            sensor_uid="TESTING",
+        )
+        with pytest.raises(CalibrationException) as e_info:
+            cal_data = cal.get_calibration_dict([100.0, 250.0])
+            assert e_info.value.args[0] == (
+                f"Could not locate calibration data at 250.0"
+                + f"\nAttempted lookup using key '250.0'"
+                + f"\nUsing calibration data: {cal.calibration_data}"
+            )
+
+    def test_sf_bound_points(self):
+        """Test SF determination at boundary points"""
+        self.run_pytest_point(
+            self.srs[0], self.frequency_min, self.gain_min, "Testing boundary points"
+        )
+        self.run_pytest_point(
+            self.srs[0], self.frequency_max, self.gain_max, "Testing boundary points"
+        )
+
+    def test_sf_no_interpolation_points(self):
+        """Test points without interpolation"""
+        for i in range(4 * self.test_repeat_times):
+            while True:
+                g = self.g_s[self.rand_index(self.g_s)]
+                f = self.f_s[self.rand_index(self.f_s)]
+                if self.run_pytest_point(
+                    self.srs[0], f, g, "Testing no interpolation points"
+                ):
+                    break
+
+    def test_update(self):
+        calibration_datetime = get_datetime_str_now()
+        calibration_params = ["sample_rate", "frequency"]
+        calibration_data = {100.0: {200.0: {"noise_figure": 0, "gain": 0}}}
+        test_cal_path = Path("test_calibration.json")
+        cal = SensorCalibration(
+            calibration_parameters=calibration_params,
+            calibration_data=calibration_data,
+            is_default=False,
+            file_path=test_cal_path,
+            last_calibration_datetime=calibration_datetime,
+            clock_rate_lookup_by_sample_rate={},
+            sensor_uid="TESTING",
+        )
+        action_params = {"sample_rate": 100.0, "frequency": 200.0}
+        update_time = get_datetime_str_now()
+        cal.update(action_params, update_time, 30.0, 5.0, 21)
+        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
+        test_cal_path.unlink()
+        file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
+        cal_time_utc = parse_datetime_iso_format_str(update_time)
+        assert file_utc_time.year == cal_time_utc.year
+        assert file_utc_time.month == cal_time_utc.month
+        assert file_utc_time.day == cal_time_utc.day
+        assert file_utc_time.hour == cal_time_utc.hour
+        assert file_utc_time.minute == cal_time_utc.minute
+        assert cal.calibration_data["100.0"]["200.0"]["gain"] == 30.0
+        assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
+        assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
+        assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
+
+    def test_filter_by_paramter_integer(self):
+        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
+        filtered_data = filter_by_parameter(calibrations, 200)
+        assert filtered_data is calibrations["200.0"]

From e4234a9840b5b2534c2dc91574ed5f7a85747bf8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:49:29 -0500
Subject: [PATCH 04/62] remove sigan calibrations

---
 .../acquire_stepped_freq_tdomain_iq.py        |  7 ------
 .../actions/interfaces/measurement_action.py  |  5 ----
 scos_actions/hardware/mocks/mock_sigan.py     | 24 ++-----------------
 scos_actions/hardware/sigan_iface.py          | 22 +----------------
 scos_actions/metadata/structs/capture.py      |  3 ++-
 5 files changed, 5 insertions(+), 56 deletions(-)

diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 495be7e2..bd114e57 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -117,14 +117,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 overload=measurement_result["overload"],
                 sigan_settings=sigan_settings,
             )
-            sigan_cal = self.sensor.signal_analyzer.sigan_calibration_data
             sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
-            if sigan_cal is not None:
-                if "1db_compression_point" in sigan_cal:
-                    sigan_cal["compression_point"] = sigan_cal.pop(
-                        "1db_compression_point"
-                    )
-                capture_segment.sigan_calibration = ntia_sensor.Calibration(**sigan_cal)
             if sensor_cal is not None:
                 if "1db_compression_point" in sensor_cal:
                     sensor_cal["compression_point"] = sensor_cal.pop(
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index f0b06e54..d3299ca4 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -51,7 +51,6 @@ def create_capture_segment(
             overload=overload,
             sigan_settings=sigan_settings,
         )
-        sigan_cal = self.sensor.signal_analyzer.sigan_calibration_data
         sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
         # Rename compression point keys if they exist
         # then set calibration metadata if it exists
@@ -61,10 +60,6 @@ def create_capture_segment(
                     "1db_compression_point"
                 )
             capture_segment.sensor_calibration = ntia_sensor.Calibration(**sensor_cal)
-        if sigan_cal is not None:
-            if "1db_compression_point" in sigan_cal:
-                sigan_cal["compression_point"] = sigan_cal.pop("1db_compression_point")
-            capture_segment.sigan_calibration = ntia_sensor.Calibration(**sigan_cal)
         return capture_segment
 
     def create_metadata(
diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 71cc71ba..35078918 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -29,20 +29,10 @@ class MockSignalAnalyzer(SignalAnalyzerInterface):
     def __init__(
         self,
         sensor_cal: Optional[SensorCalibration] = None,
-        sigan_cal: Optional[SensorCalibration] = None,
         randomize_values: bool = False,
     ):
-        super().__init__(sensor_cal, sigan_cal)
-        # Define the default calibration dicts
-        self.DEFAULT_SIGAN_CALIBRATION = {
-            "datetime": get_datetime_str_now(),
-            "gain": 0,  # Defaults to gain setting
-            "enbw": None,  # Defaults to sample rate
-            "noise_figure": 0,
-            "1db_compression_point": 100,
-            "temperature": 26.85,
-        }
-
+        super().__init__(sensor_cal)
+        # Define the default calibration dict
         self.DEFAULT_SENSOR_CALIBRATION = {
             "datetime": get_datetime_str_now(),
             "gain": 0,  # Defaults to sigan gain
@@ -73,7 +63,6 @@ def __init__(
 
         self.randomize_values = randomize_values
         self.sensor_calibration_data = self.DEFAULT_SENSOR_CALIBRATION
-        self.sigan_calibration_data = self.DEFAULT_SIGAN_CALIBRATION
 
     @property
     def is_available(self):
@@ -215,12 +204,3 @@ def recompute_sensor_calibration_data(self, cal_args: list) -> None:
             )
         else:
             logger.warning("Sensor calibration does not exist.")
-
-    def recompute_sigan_calibration_data(self, cal_args: list) -> None:
-        """Set the sigan calibration data based on the current tuning"""
-        if self.sigan_calibration is not None:
-            self.sigan_calibration_data.update(
-                self.sigan_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sigan calibration does not exist.")
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index f0eccf2c..68ed0463 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -26,13 +26,10 @@ class SignalAnalyzerInterface(ABC):
     def __init__(
         self,
         sensor_cal: Optional[SensorCalibration] = None,
-        sigan_cal: Optional[SensorCalibration] = None,
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
         self.sensor_calibration_data = {}
-        self.sigan_calibration_data = {}
         self._sensor_calibration = sensor_cal
-        self._sigan_calibration = sigan_cal
         self._model = "Unknown"
         self.switches = switches
 
@@ -136,6 +133,7 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None:
         return
 
     def recompute_sensor_calibration_data(self, cal_args: list) -> None:
+        """Set the sensor calibration data based on the current tuning."""
         self.sensor_calibration_data = {}
         if self.sensor_calibration is not None:
             self.sensor_calibration_data.update(
@@ -144,16 +142,6 @@ def recompute_sensor_calibration_data(self, cal_args: list) -> None:
         else:
             logger.warning("Sensor calibration does not exist.")
 
-    def recompute_sigan_calibration_data(self, cal_args: list) -> None:
-        self.sigan_calibration_data = {}
-        """Set the sigan calibration data based on the current tuning"""
-        if self.sigan_calibration is not None:
-            self.sigan_calibration_data.update(
-                self.sigan_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sigan calibration does not exist.")
-
     def get_status(self) -> dict:
         return {"model": self._model, "healthy": self.healthy()}
 
@@ -172,11 +160,3 @@ def sensor_calibration(self) -> SensorCalibration:
     @sensor_calibration.setter
     def sensor_calibration(self, cal: SensorCalibration):
         self._sensor_calibration = cal
-
-    @property
-    def sigan_calibration(self) -> SensorCalibration:
-        return self._sigan_calibration
-
-    @sigan_calibration.setter
-    def sigan_calibration(self, cal: SensorCalibration):
-        self._sigan_calibration = cal
diff --git a/scos_actions/metadata/structs/capture.py b/scos_actions/metadata/structs/capture.py
index 16f12280..59fe6128 100644
--- a/scos_actions/metadata/structs/capture.py
+++ b/scos_actions/metadata/structs/capture.py
@@ -1,7 +1,6 @@
 from typing import Optional
 
 import msgspec
-
 from scos_actions.metadata.structs.ntia_sensor import Calibration, SiganSettings
 from scos_actions.metadata.utils import SIGMF_OBJECT_KWARGS
 
@@ -14,6 +13,8 @@
     "duration": "ntia-sensor:duration",
     "overload": "ntia-sensor:overload",
     "sensor_calibration": "ntia-sensor:sensor_calibration",
+    # sigan_calibration is unused by SCOS Sensor but still defined
+    # in the ntia-sensor extension as of v2.0.0
     "sigan_calibration": "ntia-sensor:sigan_calibration",
     "sigan_settings": "ntia-sensor:sigan_settings",
 }

From 3be9f69bee592cd5f3a75345ce4240eb6939cb2f Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 12:51:35 -0500
Subject: [PATCH 05/62] update docstring for 'update' method

---
 scos_actions/calibration/sensor_calibration.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 11865206..d341bcab 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -33,7 +33,7 @@ def update(
         """
         Update the calibration data by overwriting or adding an entry.
 
-        This method updates the instance variables of the ``Calibration``
+        This updates the instance variables of the ``SensorCalibration``
         object and additionally writes these changes to the specified
         output file.
 
@@ -46,7 +46,6 @@ def update(
         :param noise_figure_dB: Noise figure value for calibration, in dB.
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
         :param file_path: File path for saving the updated calibration data.
-        :raises Exception:
         """
         # Get existing calibration data entry which will be updated
         data_entry = self._retrieve_data_to_update(params)

From 50b596f99252b0e8fd5d11887691e8c6bcffa0f8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 13:04:29 -0500
Subject: [PATCH 06/62] add differential calibration dataclass

---
 .../calibration/differential_calibration.py   | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 scos_actions/calibration/differential_calibration.py

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
new file mode 100644
index 00000000..cc013c6a
--- /dev/null
+++ b/scos_actions/calibration/differential_calibration.py
@@ -0,0 +1,38 @@
+"""
+Dataclass implementation for "differential calibration" handling.
+
+A differential calibration provides loss values at different frequencies
+which represent excess loss between the calibration terminal and the antenna
+port. At present, this is measured manually using a calibration probe consisting
+of a calibrated noise source and a programmable attenuator.
+
+The ``reference_point`` top-level key defines the point to which measurements
+are referenced after using the correction factors included in the file.
+
+The ``calibration_data`` entries are expected to include these correction factors,
+with the key name ``"differential_loss"`` and values in decibels (dB). These correction
+factors represent the differential loss between the calibration terminal used by onboard
+``SensorCalibration`` results and the reference point defined by ``reference_point``.
+"""
+
+from dataclasses import dataclass
+
+from scos_actions.calibration.interfaces.calibration import Calibration
+
+
+@dataclass
+class DifferentialCalibration(Calibration):
+    reference_point: str
+
+    def update(self):
+        """
+        SCOS Sensor should not update differential calibration files.
+
+        Instead, these should be generated through an external calibration
+        process. This class should only be used to read JSON files, and never
+        to update their entries. Therefore, no ``update`` method is implemented.
+
+        If, at some point in the future, functionality is added to automate these
+        calibrations, this function may be implemented.
+        """
+        raise NotImplementedError

From b8ad5d022cdee1f044901170e7484cc6590e78b8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 13:58:37 -0500
Subject: [PATCH 07/62] add field input validator to base class

---
 .../calibration/interfaces/calibration.py         | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index debe42ba..a17bc899 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -3,7 +3,7 @@
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import Any, List
+from typing import Any, List, get_origin
 
 from scos_actions.calibration.utils import filter_by_parameter
 
@@ -18,11 +18,24 @@ class Calibration:
     file_path: Path
 
     def __post_init__(self):
+        self._validate_fields()
         # Convert key names in data to strings
         # This means that formatting will always match between
         # native types provided in Python and data loaded from JSON
         self.calibration_data = json.loads(json.dumps(self.calibration_data))
 
+    def _validate_fields(self) -> None:
+        """Loosely check that the input types are as expected."""
+        for f_name, f_def in self.__dataclass_fields__.items():
+            f_type = get_origin(f_def.type) or f_def.type
+            actual_value = getattr(self, f_name)
+            if not isinstance(actual_value, f_type):
+                c_name = self.__class__.__name__
+                actual_type = type(actual_value)
+                raise TypeError(
+                    f"{c_name} field {f_name} must be {f_type}, not {actual_type}"
+                )
+
     def get_calibration_dict(self, cal_params: List[Any]) -> dict:
         """
         Get calibration data closest to the specified parameter values.

From 971409b33fbaf35c86019d9bc35095a871d2b372 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:21:26 -0500
Subject: [PATCH 08/62] add to_json method

---
 .../calibration/interfaces/calibration.py     | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index a17bc899..2e339966 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -64,6 +64,8 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
             Example: ``{"sample_rate": 14000000.0, "attenuation": 10.0}``
         :return: A dict containing the existing calibration entry at
             the specified parameter set, which may be empty if none exists.
+        :raises ValueError: If not all calibration parameters exist as keys
+            in ``params``.
         """
         # Use params keys as calibration_parameters if none exist
         if len(self.calibration_parameters) == 0:
@@ -73,7 +75,7 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
             self.calibration_parameters = list(params.keys())
         elif not set(params.keys()) >= set(self.calibration_parameters):
             # Otherwise ensure all required parameters were used
-            raise Exception(
+            raise ValueError(
                 "Not enough parameters specified to update calibration.\n"
                 + f"Required parameters are {self.calibration_parameters}"
             )
@@ -95,7 +97,7 @@ def _retrieve_data_to_update(self, params: dict) -> dict:
         return data_entry
 
     @abstractmethod
-    def update():
+    def update(self):
         """Update the calibration data"""
         raise NotImplementedError
 
@@ -141,3 +143,19 @@ def from_json(cls, fname: Path, is_default: bool):
 
         # Create and return the Calibration object
         return cls(is_default=is_default, file_path=fname, **calibration)
+
+    def to_json(self) -> None:
+        """
+        Save the calibration to a JSON file.
+
+        The JSON file will be located at ``self.file_path`` and will
+        contain a copy of ``self.__dict__``, except for the ``file_path``
+        and ``is_default`` key/value pairs. This includes all dataclass
+        fields, with their parameter names as JSON key names.
+        """
+        dict_to_json = self.__dict__.copy()
+        # Remove keys which should not save to JSON
+        dict_to_json.pop("file_path", None)
+        dict_to_json.pop("is_default", None)
+        with open(self.file_path, "w") as outfile:
+            outfile.write(json.dumps(dict_to_json))

From 96488c65a9f8e650a83633272871af1aa63c5844 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:21:55 -0500
Subject: [PATCH 09/62] use to_json in update

---
 scos_actions/calibration/sensor_calibration.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index d341bcab..57604c62 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -64,12 +64,4 @@ def update(
         )
 
         # Write updated calibration data to file
-        cal_dict = {
-            "last_calibration_datetime": self.last_calibration_datetime,
-            "sensor_uid": self.sensor_uid,
-            "calibration_parameters": self.calibration_parameters,
-            "clock_rate_lookup_by_sample_rate": self.clock_rate_lookup_by_sample_rate,
-            "calibration_data": self.calibration_data,
-        }
-        with open(self.file_path, "w") as outfile:
-            outfile.write(json.dumps(cal_dict))
+        self.to_json()

From c1a5f7d6cc1ea4c975611451419b0a5d46d829a8 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:22:38 -0500
Subject: [PATCH 10/62] update calibration-related unit tests

---
 .../calibration/tests/test_calibration.py     | 152 +++++++++++++++++-
 .../tests/test_differential_calibration.py    |  42 +++++
 .../tests/test_sensor_calibration.py          |  91 ++++++-----
 scos_actions/calibration/tests/test_utils.py  |  41 +++++
 scos_actions/calibration/tests/utils.py       |  10 ++
 5 files changed, 295 insertions(+), 41 deletions(-)
 create mode 100644 scos_actions/calibration/tests/test_differential_calibration.py
 create mode 100644 scos_actions/calibration/tests/test_utils.py
 create mode 100644 scos_actions/calibration/tests/utils.py

diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 3d743de1..a335d86f 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -1 +1,151 @@
-"""Test the Calibration base class."""
+"""Test the Calibration base dataclass."""
+import dataclasses
+import json
+from pathlib import Path
+from typing import List
+
+import pytest
+from scos_actions.calibration.interfaces.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.tests.utils import recursive_check_keys
+
+
+class TestBaseCalibration:
+    @pytest.fixture(autouse=True)
+    def setup_calibration_file(self, tmp_path: Path):
+        """Create a dummy calibration file in the pytest temp directory."""
+        # Create some dummy calibration data
+        self.cal_params = ["frequency", "gain"]
+        self.frequencies = [3555e9, 3565e9, 3575e9]
+        self.gains = [10.0, 20.0, 30.0]
+        cal_data = {}
+        for frequency in self.frequencies:
+            cal_data[frequency] = {}
+            for gain in self.gains:
+                cal_data[frequency][gain] = {
+                    "gain": gain * 1.1,
+                    "noise_figure": gain / 5.0,
+                    "1dB_compression_point": -50 + gain,
+                }
+        self.cal_data = cal_data
+        self.dummy_file_path = tmp_path / "dummy_cal.json"
+        self.dummy_default_file_path = tmp_path / "dummy_default_cal.json"
+
+        self.sample_cal = Calibration(
+            calibration_parameters=self.cal_params,
+            calibration_data=self.cal_data,
+            is_default=False,
+            file_path=self.dummy_file_path,
+        )
+
+        self.sample_default_cal = Calibration(
+            calibration_parameters=self.cal_params,
+            calibration_data=self.cal_data,
+            is_default=True,
+            file_path=self.dummy_default_file_path,
+        )
+
+    def test_calibration_data_key_name_conversion(self):
+        """On post-init, all calibration_data key names should be converted to strings."""
+        recursive_check_keys(self.sample_cal.calibration_data)
+        recursive_check_keys(self.sample_default_cal.calibration_data)
+
+    def test_calibration_dataclass_fields(self):
+        """Check that the dataclass is set up as expected."""
+        fields = {f.name: f.type for f in dataclasses.fields(Calibration)}
+        # Note: does not check field order
+        assert fields == {
+            "calibration_parameters": List[str],
+            "is_default": bool,
+            "calibration_data": dict,
+            "file_path": Path,
+        }, "Calibration class fields have changed"
+
+    def test_field_validator(self):
+        """Check that the input field type validator works."""
+        with pytest.raises(TypeError):
+            _ = Calibration([], {}, False, False)
+        with pytest.raises(TypeError):
+            _ = Calibration([], {}, 100, Path(""))
+        with pytest.raises(TypeError):
+            _ = Calibration([], [10, 20], False, Path(""))
+        with pytest.raises(TypeError):
+            _ = Calibration({"test": 1}, {}, False, Path(""))
+
+    def test_get_calibration_dict(self):
+        """Check the get_calibration_dict method with all dummy data."""
+        for f in self.frequencies:
+            assert json.loads(
+                json.dumps(self.cal_data[f])
+            ) == self.sample_cal.get_calibration_dict([f])
+            for g in self.gains:
+                assert json.loads(
+                    json.dumps(self.cal_data[f][g])
+                ) == self.sample_cal.get_calibration_dict([f, g])
+
+    def test_retrieve_data_to_update(self):
+        """Check that the calibration data entry is correctly returned."""
+        for f in self.frequencies:
+            for g in self.gains:
+                params = {"frequency": f, "gain": g}
+                # Use the "is" keyword since this must not be a copy/identical dict
+                assert self.sample_cal.calibration_data[str(f)][
+                    str(g)
+                ] is self.sample_cal._retrieve_data_to_update(params)
+        # Method should work with len=0 calibration parameters
+        test_cal = Calibration([], {}, False, Path(""))
+        _ = test_cal._retrieve_data_to_update({"frequency": 3555e9, "gain": 10.0})
+        # And should fail if calibration parameters are not all supplied
+        with pytest.raises(ValueError):
+            _ = self.sample_cal._retrieve_data_to_update(
+                {"frequency": self.frequencies[0]}
+            )
+
+    def test_to_and_from_json(self, tmp_path: Path):
+        """Test the ``from_json`` factory method."""
+        # First save the calibration data to temporary files
+        self.sample_cal.to_json()
+        self.sample_default_cal.to_json()
+        # Then load and compare
+        assert self.sample_cal == Calibration.from_json(self.dummy_file_path, False)
+        assert self.sample_default_cal == Calibration.from_json(
+            self.dummy_default_file_path, True
+        )
+
+        # These should fail: the is_default parameter is different
+        # even though the other contents are identical.
+        with pytest.raises(AssertionError):
+            assert self.sample_cal == Calibration.from_json(self.dummy_file_path, True)
+        with pytest.raises(AssertionError):
+            assert self.sample_default_cal == Calibration.from_json(
+                self.dummy_default_file_path, False
+            )
+
+        # from_json should ignore extra keys in the loaded file, but not fail
+        # Test this by trying to load a SensorCalibration as a Calibration
+        sensor_cal = SensorCalibration(
+            self.sample_cal.calibration_parameters,
+            self.sample_cal.calibration_data,
+            False,
+            tmp_path / "testing.json",
+            "dt_str",
+            [],
+            "uid",
+        )
+        sensor_cal.to_json()
+        loaded_cal = Calibration.from_json(tmp_path / "testing.json", False)
+        loaded_cal.file_path = self.sample_cal.file_path  # Force these to be the same
+        assert loaded_cal == self.sample_cal
+
+        # from_json should fail if required fields are missing
+        # Create an incorrect JSON file
+        almost_a_cal = {"calibration_parameters": []}
+        with open(tmp_path / "almost_a_cal.json", "w") as outfile:
+            outfile.write(json.dumps(almost_a_cal))
+        with pytest.raises(Exception):
+            almost = Calibration.from_json(tmp_path / "almost_a_cal.json", False)
+
+    def test_update_not_implemented(self):
+        """Ensure the update abstract method is not implemented in the base class"""
+        with pytest.raises(NotImplementedError):
+            self.sample_cal.update()
diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
new file mode 100644
index 00000000..8f1b5594
--- /dev/null
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -0,0 +1,42 @@
+"""Test the DifferentialCalibration dataclass."""
+import json
+from pathlib import Path
+
+import pytest
+from scos_actions.calibration.differential_calibration import DifferentialCalibration
+
+
+class TestDifferentialCalibration:
+    @pytest.fixture(autouse=True)
+    def setup_differential_calibration_file(self, tmp_path: Path):
+        dict_to_json = {
+            "calibration_parameters": ["frequency"],
+            "reference_point": "antenna input",
+            "calibration_data": {3555e6: 11.5},
+        }
+        self.valid_file_path = tmp_path / "sample_diff_cal.json"
+        self.invalid_file_path = tmp_path / "sample_diff_cal_invalid.json"
+
+        self.sample_diff_cal = DifferentialCalibration(
+            is_default=False, file_path=self.valid_file_path, **dict_to_json
+        )
+
+        with open(self.valid_file_path, "w") as f:
+            f.write(json.dumps(dict_to_json))
+
+        dict_to_json.pop("reference_point", None)
+
+        with open(self.invalid_file_path, "w") as f:
+            f.write(json.dumps(dict_to_json))
+
+    def test_from_json(self):
+        """Check from_json functionality with valid and invalid dummy data."""
+        diff_cal = DifferentialCalibration.from_json(self.valid_file_path, False)
+        assert diff_cal == self.sample_diff_cal
+        with pytest.raises(Exception):
+            _ = DifferentialCalibration.from_json(self.invalid_file_path, False)
+
+    def test_update_not_implemented(self):
+        """Check that the update method is not implemented."""
+        with pytest.raises(NotImplementedError):
+            self.sample_diff_cal.update()
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index cf4403fa..6348c409 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -1,14 +1,18 @@
-"""Test the SensorCalibration class."""
-
+"""Test the SensorCalibration dataclass."""
+import dataclasses
+import datetime
 import json
 import random
 from copy import deepcopy
 from math import isclose
 from pathlib import Path
+from typing import Dict, List
 
 import pytest
+from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
-from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
+from scos_actions.calibration.tests.utils import recursive_check_keys
+from scos_actions.calibration.utils import CalibrationException
 from scos_actions.tests.resources.utils import easy_gain
 from scos_actions.utils import get_datetime_str_now, parse_datetime_iso_format_str
 
@@ -93,7 +97,7 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
         return True
 
     @pytest.fixture(autouse=True)
-    def setup_calibration_file(self, tmpdir):
+    def setup_calibration_file(self, tmp_path: Path):
         """Create the dummy calibration file in the pytest temp directory"""
 
         # Only setup once
@@ -101,8 +105,7 @@ def setup_calibration_file(self, tmpdir):
             return
 
         # Create and save the temp directory and file
-        self.tmpdir = tmpdir.strpath
-        self.calibration_file = "{}".format(tmpdir.join("dummy_cal_file.json"))
+        self.calibration_file = tmp_path / "dummy_cal_file.json"
 
         # Setup variables
         self.dummy_noise_figure = 10
@@ -191,30 +194,43 @@ def setup_calibration_file(self, tmpdir):
         # Don't repeat test setup
         self.setup_complete = True
 
-    def test_filter_by_parameter_out_of_range(self):
-        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 400.0)
-            assert (
-                e_info.value.args[0]
-                == f"Could not locate calibration data at 400.0"
-                + f"\nAttempted lookup using key '400.0'"
-                + f"\nUsing calibration data: {calibrations}"
-            )
+    def test_calibration_data_key_name_conversion(self):
+        """On post-init, all calibration_data key names should be converted to strings."""
+        recursive_check_keys(self.sample_cal.calibration_data)
 
-    def test_filter_by_parameter_in_range_requires_match(self):
-        calibrations = {
-            200.0: {"Gain": "Gain at 200.0"},
-            300.0: {"Gain": "Gain at 300.0"},
+    def test_sensor_calibration_dataclass_fields(self):
+        """Check that the dataclass fields are as expected."""
+        fields = {
+            f.name: f.type
+            for f in dataclasses.fields(SensorCalibration)
+            if f not in dataclasses.fields(Calibration)
         }
-        with pytest.raises(CalibrationException) as e_info:
-            cal = filter_by_parameter(calibrations, 150.0)
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 150.0"
-                + f"\nAttempted lookup using key '150.0'"
-                + f"\nUsing calibration data: {calibrations}"
+        # Note: does not check field order
+        assert fields == {
+            "last_calibration_datetime": str,
+            "clock_rate_lookup_by_sample_rate": List[Dict[str, float]],
+            "sensor_uid": str,
+        }
+
+    def test_field_validator(self):
+        """Check that the input field type validator works."""
+        # only check fields not inherited from Calibration base class
+        with pytest.raises(TypeError):
+            _ = SensorCalibration([], {}, False, Path(""), "dt_str", [], 10)
+        with pytest.raises(TypeError):
+            _ = SensorCalibration([], {}, False, Path(""), "dt_str", {}, "uid")
+        with pytest.raises(TypeError):
+            _ = SensorCalibration(
+                [], {}, False, Path(""), datetime.datetime.now(), [], "uid"
             )
 
+    def test_get_clock_rate(self):
+        """Test the get_clock_rate method"""
+        # Test getting a clock rate by sample rate
+        assert self.sample_cal.get_clock_rate(10e6) == 40e6
+        # If there isn't an entry, the sample rate should be returned
+        assert self.sample_cal.get_clock_rate(-999) == -999
+
     def test_get_calibration_dict_exact_match_lookup(self):
         calibration_datetime = get_datetime_str_now()
         calibration_params = ["sample_rate", "frequency"]
@@ -228,7 +244,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
             is_default=False,
             file_path=Path(""),
             last_calibration_datetime=calibration_datetime,
-            clock_rate_lookup_by_sample_rate={},
+            clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
         cal_data = cal.get_calibration_dict([100.0, 200.0])
@@ -247,16 +263,16 @@ def test_get_calibration_dict_within_range(self):
             is_default=False,
             file_path=Path("test_calibration.json"),
             last_calibration_datetime=calibration_datetime,
-            clock_rate_lookup_by_sample_rate={},
+            clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
         with pytest.raises(CalibrationException) as e_info:
-            cal_data = cal.get_calibration_dict([100.0, 250.0])
-            assert e_info.value.args[0] == (
-                f"Could not locate calibration data at 250.0"
-                + f"\nAttempted lookup using key '250.0'"
-                + f"\nUsing calibration data: {cal.calibration_data}"
-            )
+            _ = cal.get_calibration_dict([100.0, 250.0])
+        assert e_info.value.args[0] == (
+            f"Could not locate calibration data at 250.0"
+            + f"\nAttempted lookup using key '250.0'"
+            + f"\nUsing calibration data: {cal.calibration_data['100.0']}"
+        )
 
     def test_sf_bound_points(self):
         """Test SF determination at boundary points"""
@@ -289,7 +305,7 @@ def test_update(self):
             is_default=False,
             file_path=test_cal_path,
             last_calibration_datetime=calibration_datetime,
-            clock_rate_lookup_by_sample_rate={},
+            clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
         action_params = {"sample_rate": 100.0, "frequency": 200.0}
@@ -308,8 +324,3 @@ def test_update(self):
         assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
-
-    def test_filter_by_paramter_integer(self):
-        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
-        filtered_data = filter_by_parameter(calibrations, 200)
-        assert filtered_data is calibrations["200.0"]
diff --git a/scos_actions/calibration/tests/test_utils.py b/scos_actions/calibration/tests/test_utils.py
new file mode 100644
index 00000000..2a751831
--- /dev/null
+++ b/scos_actions/calibration/tests/test_utils.py
@@ -0,0 +1,41 @@
+import pytest
+from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
+
+
+class TestCalibrationUtils:
+    def test_filter_by_parameter_out_of_range(self):
+        calibrations = {200.0: {"some_cal_data"}, 300.0: {"more cal data"}}
+        with pytest.raises(CalibrationException) as e_info:
+            _ = filter_by_parameter(calibrations, 400.0)
+        assert (
+            e_info.value.args[0]
+            == f"Could not locate calibration data at 400.0"
+            + f"\nAttempted lookup using key '400.0'"
+            + f"\nUsing calibration data: {calibrations}"
+        )
+
+    def test_filter_by_parameter_in_range_requires_match(self):
+        calibrations = {
+            200.0: {"Gain": "Gain at 200.0"},
+            300.0: {"Gain": "Gain at 300.0"},
+        }
+        with pytest.raises(CalibrationException) as e_info:
+            _ = filter_by_parameter(calibrations, 150.0)
+        assert e_info.value.args[0] == (
+            f"Could not locate calibration data at 150.0"
+            + f"\nAttempted lookup using key '150.0'"
+            + f"\nUsing calibration data: {calibrations}"
+        )
+
+    def test_filter_by_paramter_integer(self):
+        calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}}
+        filtered_data = filter_by_parameter(calibrations, 200)
+        assert filtered_data is calibrations["200.0"]
+
+    def test_filter_by_parameter_type_error(self):
+        calibrations = [300.0, 400.0]
+        with pytest.raises(CalibrationException) as e_info:
+            _ = filter_by_parameter(calibrations, 300.0)
+        assert e_info.value.args[0] == (
+            f"Provided calibration data is not a dict: {calibrations}"
+        )
diff --git a/scos_actions/calibration/tests/utils.py b/scos_actions/calibration/tests/utils.py
new file mode 100644
index 00000000..a2ad6e51
--- /dev/null
+++ b/scos_actions/calibration/tests/utils.py
@@ -0,0 +1,10 @@
+"""Utility functions used for scos_sensor.calibration unit tests."""
+
+
+def recursive_check_keys(d: dict):
+    """Recursively checks a dict to check that all keys are strings"""
+    for k, v in d.items():
+        if isinstance(v, dict):
+            recursive_check_keys(v)
+        else:
+            assert isinstance(k, str)

From c304e0207c412141e0d81682382d7c7a21891bff Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 15:33:32 -0500
Subject: [PATCH 11/62] bump version number

---
 scos_actions/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py
index 73d4c8be..6dcf770e 100644
--- a/scos_actions/__init__.py
+++ b/scos_actions/__init__.py
@@ -1 +1 @@
-__version__ = "8.0.0"
+__version__ = "9.0.0"

From fc2d450b6adba1a494023c9918a56484fb6f9d76 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 22 Jan 2024 16:07:11 -0500
Subject: [PATCH 12/62] remove unused imports

---
 scos_actions/calibration/sensor_calibration.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 57604c62..12997f22 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -1,7 +1,5 @@
-import json
 import logging
 from dataclasses import dataclass
-from pathlib import Path
 from typing import Dict, List, Union
 
 from scos_actions.calibration.interfaces.calibration import Calibration

From 013b791e7e1d9db666bdcc7c8d6a762625cc83b7 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 10:37:04 -0500
Subject: [PATCH 13/62] remove unused imports

---
 scos_actions/hardware/utils.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scos_actions/hardware/utils.py b/scos_actions/hardware/utils.py
index d9f6fbf2..b479f2a5 100644
--- a/scos_actions/hardware/utils.py
+++ b/scos_actions/hardware/utils.py
@@ -1,14 +1,9 @@
 import logging
 import subprocess
-from pathlib import Path
 from typing import Dict
 
 import psutil
-from its_preselector.configuration_exception import ConfigurationException
-from its_preselector.controlbyweb_web_relay import ControlByWebWebRelay
 from its_preselector.web_relay import WebRelay
-
-from scos_actions import utils
 from scos_actions.hardware.hardware_configuration_exception import (
     HardwareConfigurationException,
 )

From ddac9c5ce322b26da55d38467c6a8698ea1bba4a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 20:05:53 -0500
Subject: [PATCH 14/62] simplify expressions in Y-factor cal

Avoids some redundant additions and subtractions by working internally in dBW instead of dBm
---
 scos_actions/signal_processing/calibration.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py
index 757dfc50..3f98ade8 100644
--- a/scos_actions/signal_processing/calibration.py
+++ b/scos_actions/signal_processing/calibration.py
@@ -11,7 +11,6 @@
     convert_celsius_to_kelvins,
     convert_dB_to_linear,
     convert_linear_to_dB,
-    convert_watts_to_dBm,
 )
 
 logger = logging.getLogger(__name__)
@@ -43,16 +42,16 @@ def y_factor(
     :return: A tuple (noise_figure, gain) containing the calculated
         noise figure and gain, both in dB, from the Y-factor method.
     """
-    mean_on_dBm = convert_watts_to_dBm(np.mean(pwr_noise_on_watts))
-    mean_off_dBm = convert_watts_to_dBm(np.mean(pwr_noise_off_watts))
+    mean_on_dBW = convert_linear_to_dB(np.mean(pwr_noise_on_watts))
+    mean_off_dBW = convert_linear_to_dB(np.mean(pwr_noise_off_watts))
     if logger.isEnabledFor(logging.DEBUG):
         logger.debug(f"ENR: {convert_linear_to_dB(enr_linear)} dB")
         logger.debug(f"ENBW: {enbw_hz} Hz")
-        logger.debug(f"Mean power on: {mean_on_dBm:.2f} dBm")
-        logger.debug(f"Mean power off: {mean_off_dBm:.2f} dBm")
-    y = convert_dB_to_linear(mean_on_dBm - mean_off_dBm)
+        logger.debug(f"Mean power on: {mean_on_dBW+30:.2f} dBm")
+        logger.debug(f"Mean power off: {mean_off_dBW+30:.2f} dBm")
+    y = convert_dB_to_linear(mean_on_dBW - mean_off_dBW)
     noise_factor = enr_linear / (y - 1.0)
-    gain_dB = mean_on_dBm - convert_watts_to_dBm(
+    gain_dB = mean_on_dBW - convert_linear_to_dB(
         Boltzmann * temp_kelvins * enbw_hz * (enr_linear + noise_factor)
     )
     noise_figure_dB = convert_linear_to_dB(noise_factor)

From f978515e097bafa6f0771dbff7f79bd4c08ed199 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 20:23:48 -0500
Subject: [PATCH 15/62] Use "loss" as value key instead of "differential_loss"

---
 scos_actions/calibration/differential_calibration.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
index cc013c6a..4cf40322 100644
--- a/scos_actions/calibration/differential_calibration.py
+++ b/scos_actions/calibration/differential_calibration.py
@@ -10,9 +10,11 @@
 are referenced after using the correction factors included in the file.
 
 The ``calibration_data`` entries are expected to include these correction factors,
-with the key name ``"differential_loss"`` and values in decibels (dB). These correction
-factors represent the differential loss between the calibration terminal used by onboard
+with the key name ``"loss"`` and values in decibels (dB). These correction factors
+represent the differential loss between the calibration terminal used by onboard
 ``SensorCalibration`` results and the reference point defined by ``reference_point``.
+A positive value of ``"loss"`` indicates a LOSS going FROM ``reference_point`` TO
+the calibration terminal used by the ``SensorCalibration``.
 """
 
 from dataclasses import dataclass

From 0832db6e2babe647d0ecc2b2b42d6d4e47f3ee12 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 20:24:51 -0500
Subject: [PATCH 16/62] clarify functionality of _validate_fields

---
 scos_actions/calibration/interfaces/calibration.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 2e339966..406e325e 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -27,6 +27,8 @@ def __post_init__(self):
     def _validate_fields(self) -> None:
         """Loosely check that the input types are as expected."""
         for f_name, f_def in self.__dataclass_fields__.items():
+            # Note that nested types are not checked: i.e., "List[str]"
+            # will surely be a list, but may not be filled with strings.
             f_type = get_origin(f_def.type) or f_def.type
             actual_value = getattr(self, f_name)
             if not isinstance(actual_value, f_type):

From 49503f71903ed23fa2231fd78e41375b17ffffda Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 21:15:26 -0500
Subject: [PATCH 17/62] implement sensor-level acquire_samples

- fully remove calibration from sigan interfaces. all calibration handling happens at the sensor object level
- alter the sigan interface: retry logic and calibration scaling now happens in the sensor acquire_samples method instead of sigan versions.
- dynamically set data reference point as part the measurement result based on which calibration scaling was applied
---
 scos_actions/hardware/mocks/mock_sigan.py |  19 +-
 scos_actions/hardware/sensor.py           | 214 +++++++++++++++++++---
 scos_actions/hardware/sigan_iface.py      |  41 +----
 3 files changed, 197 insertions(+), 77 deletions(-)

diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 2fdc04df..03a9e42b 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -4,7 +4,6 @@
 from typing import Optional
 
 import numpy as np
-from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import get_datetime_str_now
 
@@ -28,11 +27,10 @@ class MockSignalAnalyzer(SignalAnalyzerInterface):
 
     def __init__(
         self,
-        sensor_cal: Optional[SensorCalibration] = None,
         switches: Optional[dict] = None,
         randomize_values: bool = False,
     ):
-        super().__init__(sensor_cal)
+        super().__init__(switches)
         # Define the default calibration dict
         self.DEFAULT_SENSOR_CALIBRATION = {
             "datetime": get_datetime_str_now(),
@@ -136,8 +134,8 @@ def connect(self):
         pass
 
     def acquire_time_domain_samples(
-        self, num_samples, num_samples_skip=0, retries=5, cal_adjust=True
-    ):
+        self, num_samples: int, num_samples_skip: int = 0, retries: int = 5
+    ) -> dict:
         logger.warning("Using mock signal analyzer!")
         self.sigan_overload = False
         self._capture_time = None
@@ -194,14 +192,3 @@ def set_times_to_fail_recv(self, n):
     @property
     def last_calibration_time(self):
         return get_datetime_str_now()
-
-    def update_calibration(self, params):
-        pass
-
-    def recompute_sensor_calibration_data(self, cal_args: list) -> None:
-        if self.sensor_calibration is not None:
-            self.sensor_calibration_data.update(
-                self._sensor_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sensor calibration does not exist.")
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index c4120a33..91f9a273 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -1,25 +1,35 @@
 import datetime
 import hashlib
 import json
-from typing import Dict
+import logging
+from typing import Dict, Optional
 
 from its_preselector.preselector import Preselector
 from its_preselector.web_relay import WebRelay
+from scos_actions.calibration.differential_calibration import DifferentialCalibration
+from scos_actions.calibration.interfaces.calibration import Calibration
+from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.hardware.gps_iface import GPSInterface
+from scos_actions.hardware.sigan_iface import (
+    SIGAN_SETTINGS_KEYS,
+    SignalAnalyzerInterface,
+)
+from scos_actions.utils import convert_string_to_millisecond_iso_format
 
-from .gps_iface import GPSInterface
-from .mocks.mock_sigan import MockSignalAnalyzer
-from .sigan_iface import SignalAnalyzerInterface
+logger = logging.getLogger(__name__)
 
 
 class Sensor:
     def __init__(
         self,
-        signal_analyzer: SignalAnalyzerInterface = None,
-        gps: GPSInterface = None,
-        preselector: Preselector = None,
+        signal_analyzer: Optional[SignalAnalyzerInterface] = None,
+        gps: Optional[GPSInterface] = None,
+        preselector: Optional[Preselector] = None,
         switches: Dict[str, WebRelay] = {},
-        location: dict = None,
-        capabilities: dict = None,
+        location: Optional[dict] = None,
+        capabilities: Optional[dict] = None,
+        sensor_cal: Optional[SensorCalibration] = None,
+        differential_cal: Optional[DifferentialCalibration] = None,
     ):
         self.signal_analyzer = signal_analyzer
         self.gps = gps
@@ -27,31 +37,34 @@ def __init__(
         self.switches = switches
         self.location = location
         self.capabilities = capabilities
+        self.sensor_calibration_data = {}
+        self._sensor_calibration = sensor_cal
+        self._differential_calibration = differential_cal
         # There is no setter for start_time property
         self._start_time = datetime.datetime.utcnow()
 
     @property
-    def signal_analyzer(self) -> SignalAnalyzerInterface:
+    def signal_analyzer(self) -> Optional[SignalAnalyzerInterface]:
         return self._signal_analyzer
 
     @signal_analyzer.setter
-    def signal_analyzer(self, sigan: SignalAnalyzerInterface):
+    def signal_analyzer(self, sigan: Optional[SignalAnalyzerInterface]):
         self._signal_analyzer = sigan
 
     @property
-    def gps(self) -> GPSInterface:
+    def gps(self) -> Optional[GPSInterface]:
         return self._gps
 
     @gps.setter
-    def gps(self, gps: GPSInterface):
+    def gps(self, gps: Optional[GPSInterface]):
         self._gps = gps
 
     @property
-    def preselector(self) -> Preselector:
+    def preselector(self) -> Optional[Preselector]:
         return self._preselector
 
     @preselector.setter
-    def preselector(self, preselector: Preselector):
+    def preselector(self, preselector: Optional[Preselector]):
         self._preselector = preselector
 
     @property
@@ -63,19 +76,19 @@ def switches(self, switches: Dict[str, WebRelay]):
         self._switches = switches
 
     @property
-    def location(self) -> dict:
+    def location(self) -> Optional[dict]:
         return self._location
 
     @location.setter
-    def location(self, loc: dict):
+    def location(self, loc: Optional[dict]):
         self._location = loc
 
     @property
-    def capabilities(self) -> dict:
+    def capabilities(self) -> Optional[dict]:
         return self._capabilities
 
     @capabilities.setter
-    def capabilities(self, capabilities: dict):
+    def capabilities(self, capabilities: Optional[dict]):
         if capabilities is not None:
             if "sensor_sha512" not in capabilities["sensor"]:
                 sensor_def = json.dumps(capabilities["sensor"], sort_keys=True)
@@ -83,9 +96,7 @@ def capabilities(self, capabilities: dict):
                     sensor_def.encode("UTF-8")
                 ).hexdigest()
                 capabilities["sensor"]["sensor_sha512"] = SENSOR_DEFINITION_HASH
-            self._capabilities = capabilities
-        else:
-            self._capabilities = None
+        self._capabilities = capabilities
 
     @property
     def has_configurable_preselector(self) -> bool:
@@ -102,5 +113,162 @@ def has_configurable_preselector(self) -> bool:
                 return False
 
     @property
-    def start_time(self):
+    def start_time(self) -> datetime.datetime:
         return self._start_time
+
+    @property
+    def sensor_calibration(self) -> Optional[SensorCalibration]:
+        return self._sensor_calibration
+
+    @sensor_calibration.setter
+    def sensor_calibration(self, cal: Optional[SensorCalibration]):
+        self._sensor_calibration = cal
+
+    @property
+    def differential_calibration(self) -> Optional[DifferentialCalibration]:
+        return self._differential_calibration
+
+    @differential_calibration.setter
+    def differential_calibration(self, cal: Optional[DifferentialCalibration]):
+        self._differential_calibration = cal
+
+    @property
+    def last_calibration_time(self) -> str:
+        """Get a datetime string for the most recent sensor calibration."""
+        return convert_string_to_millisecond_iso_format(
+            self.sensor_calibration.last_calibration_datetime
+        )
+
+    def _get_calibration_args_from_sigan(self, calibration: Calibration) -> list:
+        """Get current values of signal analyzer settings which are calibration parameters."""
+        cal_params = [
+            p for p in calibration.calibration_parameters if p in SIGAN_SETTINGS_KEYS
+        ]
+        if set(cal_params) <= set(calibration.calibration_parameters):
+            msg = (
+                "One or more required calibration parameters is not a valid signal "
+                + f"analyzer property.\nRequired parameters: {calibration.calibration_parameters}"
+                + f"\nSignal analyzer properties: {list(vars(self.signal_analyzer).keys())}"
+            )
+            logger.error(msg)
+            raise KeyError
+        cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
+        logger.debug(f"Matched calibration params: {cal_args}")
+        return cal_args  # Order matches calibration.calibration_parameters
+
+    def recompute_differential_calibration_data(self) -> None:
+        """Set the differential calibration data based on the current tuning."""
+        self.differential_calibration_data = {}
+        if self.differential_calibration is not None:
+            cal_args = self._get_calibration_args_from_sigan(
+                self.differential_calibration
+            )
+            self.differential_calibration_data.update(
+                self.differential_calibration.get_calibration_dict(cal_args)
+            )
+        else:
+            logger.warning("Differential calibration does not exist.")
+
+    def recompute_sensor_calibration_data(self) -> None:
+        """Set the sensor calibration data based on the current tuning."""
+        self.sensor_calibration_data = {}
+        if self.sensor_calibration is not None:
+            cal_args = self._get_calibration_args_from_sigan(self.sensor_calibration)
+            self.sensor_calibration_data.update(
+                self.sensor_calibration.get_calibration_dict(cal_args)
+            )
+        else:
+            logger.warning("Sensor calibration does not exist.")
+
+    def acquire_time_domain_samples(
+        self,
+        num_samples: int,
+        num_samples_skip: int = 0,
+        retries: int = 5,
+        cal_adjust: bool = True,
+    ) -> dict:
+        """
+        Acquire time-domain IQ samples from the signal analyzer.
+
+        Signal analyzer settings, preselector state, etc. should already be
+        set before calling this function.
+
+        Gain adjustment can be applied to acquired samples using ``cal_adjust``.
+        If ``True``, the samples acquired from the signal analyzer will be
+        scaled based on the calibrated ``gain`` value in the ``SensorCalibration``,
+        if one exists for this sensor, and "calibration terminal" will be the value
+        of the "reference" key in the returned dict.
+
+        :param num_samples: Number of samples to acquire
+        :param num_samples_skip: Number of samples to skip
+        :param retries: Maximum number of retries on failure
+        :param cal_adjust: If True, use available calibration data to scale the samples.
+        :return: dictionary containing data, sample_rate, frequency, capture_time, etc
+        :raises Exception: If the sample acquisition fails, or the sensor has
+            no signal analyzer.
+        """
+        logger.debug("Sensor.acquire_time_domain_samples starting")
+        logger.debug(f"Number of retries = {retries}")
+        max_retries = retries
+        # TODO: Include RF path as a sensor cal argument?
+        # Acquire samples from signal analyzer
+        if self.signal_analyzer is not None:
+            while True:
+                try:
+                    measurement_result = (
+                        self.signal_analyzer.acquire_time_domain_samples(
+                            num_samples, num_samples_skip
+                        )
+                    )
+                    break
+                except Exception as e:
+                    retries -= 1
+                    logger.info("Error while acquiring samples from signal analyzer.")
+                    if retries == 0:
+                        logger.exception(
+                            "Failed to acquire samples from signal analyzer. "
+                            + f"Tried {max_retries} times."
+                        )
+                        raise e
+        else:
+            msg = "Failed to acquire samples: sensor has no signal analyzer"
+            logger.error(msg)
+            raise Exception(msg)
+
+        # Apply gain adjustment based on calibration
+        if cal_adjust:
+            if self.sensor_calibration is not None:
+                logger.debug("Scaling samples using calibration data")
+                calibrated_gain__db = 0.0
+                self.recompute_sensor_calibration_data()
+                sensor_gain = self.sensor_calibration_data["gain"]
+                logger.debug(f"Using sensor gain: {sensor_gain} dB")
+                calibrated_gain__db += sensor_gain
+                if self.differential_calibration is not None:
+                    # Also apply differential calibration correction
+                    # TODO recompute functions match to current signal analyzer
+                    # settings. should they use the measurement_result instead?
+                    self.recompute_differential_calibration_data()
+                    differential_loss = self.differential_calibration_data["loss"]
+                    logger.debug(f"Using differential loss: {differential_loss} dB")
+                    calibrated_gain__db -= differential_loss
+                    measurement_result[
+                        "reference"
+                    ] = self.differential_calibration.reference_point
+                else:
+                    # No differential calibration exists
+                    logger.debug("No differential calibration was applied")
+                    measurement_result["reference"] = "calibration terminal"
+                linear_gain = 10.0 ** (calibrated_gain__db / 20.0)
+                logger.debug(f"Applying gain of {linear_gain}")
+                measurement_result["data"] /= linear_gain
+            else:
+                # No sensor calibration exists
+                msg = "Unable to scale samples without sensor calibration data"
+                logger.error(msg)
+                raise Exception(msg)
+        else:
+            # Set the data reference in the measurement_result
+            measurement_result["reference"] = "signal analyzer input"
+
+        return measurement_result
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index 68ed0463..f7fb0bed 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -4,9 +4,7 @@
 from typing import Dict, Optional
 
 from its_preselector.web_relay import WebRelay
-from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.utils import power_cycle_sigan
-from scos_actions.utils import convert_string_to_millisecond_iso_format
 
 logger = logging.getLogger(__name__)
 
@@ -25,21 +23,11 @@
 class SignalAnalyzerInterface(ABC):
     def __init__(
         self,
-        sensor_cal: Optional[SensorCalibration] = None,
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
-        self.sensor_calibration_data = {}
-        self._sensor_calibration = sensor_cal
         self._model = "Unknown"
         self.switches = switches
 
-    @property
-    def last_calibration_time(self) -> str:
-        """Returns the last calibration time from calibration data."""
-        return convert_string_to_millisecond_iso_format(
-            self.sensor_calibration.last_calibration_datetime
-        )
-
     @property
     @abstractmethod
     def is_available(self) -> bool:
@@ -67,16 +55,13 @@ def acquire_time_domain_samples(
         self,
         num_samples: int,
         num_samples_skip: int = 0,
-        retries: int = 5,
-        cal_adjust: bool = True,
     ) -> dict:
         """
-        Acquire time domain IQ samples
+        Acquire time domain IQ samples, scaled to Volts at
+        the signal analyzer input.
 
         :param num_samples: Number of samples to acquire
         :param num_samples_skip: Number of samples to skip
-        :param retries: Maximum number of retries on failure
-        :param cal_adjust: If True, scale IQ samples based on calibration data.
         :return: dictionary containing data, sample_rate, frequency, capture_time, etc
         """
         pass
@@ -94,9 +79,7 @@ def healthy(self, num_samples: int = 56000) -> bool:
         if not self.is_available:
             return False
         try:
-            measurement_result = self.acquire_time_domain_samples(
-                num_samples, cal_adjust=False
-            )
+            measurement_result = self.acquire_time_domain_samples(num_samples)
             data = measurement_result["data"]
         except Exception as e:
             logger.exception("Unable to acquire samples from device.")
@@ -132,16 +115,6 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None:
             )
         return
 
-    def recompute_sensor_calibration_data(self, cal_args: list) -> None:
-        """Set the sensor calibration data based on the current tuning."""
-        self.sensor_calibration_data = {}
-        if self.sensor_calibration is not None:
-            self.sensor_calibration_data.update(
-                self.sensor_calibration.get_calibration_dict(cal_args)
-            )
-        else:
-            logger.warning("Sensor calibration does not exist.")
-
     def get_status(self) -> dict:
         return {"model": self._model, "healthy": self.healthy()}
 
@@ -152,11 +125,3 @@ def model(self) -> str:
     @model.setter
     def model(self, value: str):
         self._model = value
-
-    @property
-    def sensor_calibration(self) -> SensorCalibration:
-        return self._sensor_calibration
-
-    @sensor_calibration.setter
-    def sensor_calibration(self, cal: SensorCalibration):
-        self._sensor_calibration = cal

From 4a81a2ad9e39e748e94a177078122a6f523807a2 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 21:18:12 -0500
Subject: [PATCH 18/62] remove unused import

---
 scos_actions/settings.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scos_actions/settings.py b/scos_actions/settings.py
index fa63aaa1..7d45d179 100644
--- a/scos_actions/settings.py
+++ b/scos_actions/settings.py
@@ -1,5 +1,4 @@
 import logging
-from os import path
 from pathlib import Path
 
 from environs import Env

From 43ac4be377d9141c9a60b789d4dd870df7fbc7f9 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 23 Jan 2024 21:50:07 -0500
Subject: [PATCH 19/62] update actions for sensor cal handling changes

---
 .../actions/acquire_sea_data_product.py       | 33 ++++++++++---------
 .../actions/acquire_single_freq_fft.py        |  8 ++---
 .../actions/acquire_single_freq_tdomain_iq.py |  2 +-
 .../acquire_stepped_freq_tdomain_iq.py        | 11 ++++---
 scos_actions/actions/calibrate_y_factor.py    | 26 ++++++---------
 .../actions/interfaces/measurement_action.py  |  4 +--
 scos_actions/hardware/sensor.py               | 19 ++++++++---
 7 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 3409699a..003db3b8 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -34,9 +34,6 @@
 from environs import Env
 from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
-
-from scos_actions import __version__ as SCOS_ACTIONS_VERSION
-from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.utils import (
@@ -76,6 +73,9 @@
 from scos_actions.signals import measurement_action_completed, trigger_api_restart
 from scos_actions.utils import convert_datetime_to_millisecond_iso_format, get_days_up
 
+from scos_actions import __version__ as SCOS_ACTIONS_VERSION
+from scos_actions import utils
+
 env = Env()
 logger = logging.getLogger(__name__)
 
@@ -112,7 +112,7 @@
 FFT_WINDOW = get_fft_window(FFT_WINDOW_TYPE, FFT_SIZE)
 FFT_WINDOW_ECF = get_fft_window_correction(FFT_WINDOW, "energy")
 IMPEDANCE_OHMS = 50.0
-DATA_REFERENCE_POINT = "noise source output"
+DATA_REFERENCE_POINT = "noise source output"  # TODO delete
 NUM_ACTORS = 3  # Number of ray actors to initialize
 
 # Create power detectors
@@ -626,14 +626,13 @@ def capture_iq(self, params: dict) -> dict:
         nskip = utils.get_parameter(NUM_SKIP, params)
         num_samples = int(params[SAMPLE_RATE] * duration_ms * 1e-3)
         # Collect IQ data
-        measurement_result = self.sensor.signal_analyzer.acquire_time_domain_samples(
-            num_samples, nskip
-        )
+        measurement_result = self.sensor.acquire_time_domain_samples(num_samples, nskip)
         # Store some metadata with the IQ
         measurement_result.update(params)
+        measurement_result["sensor_cal"] = self.sensor.sensor_calibration_data
         measurement_result[
-            "sensor_cal"
-        ] = self.sensor.signal_analyzer.sensor_calibration_data
+            "differential_cal"
+        ] = self.sensor.differential_calibration_data
         toc = perf_counter()
         logger.debug(
             f"IQ Capture ({duration_ms} ms @ {(params[FREQUENCY]/1e6):.1f} MHz) completed in {toc-tic:.2f} s."
@@ -1019,7 +1018,7 @@ def create_global_data_product_metadata(self) -> None:
             x_step=[p[SAMPLE_RATE] / FFT_SIZE],
             y_units="dBm/Hz",
             processing=[dft_obj.id],
-            reference=DATA_REFERENCE_POINT,
+            reference=DATA_REFERENCE_POINT,  # TODO update
             description=(
                 "Results of statistical detectors (max, mean, median, 25th_percentile, 75th_percentile, "
                 + "90th_percentile, 95th_percentile, 99th_percentile, 99.9th_percentile, 99.99th_percentile) "
@@ -1039,7 +1038,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pvt_x_axis__s[-1]],
             x_step=[pvt_x_axis__s[1] - pvt_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,
+            reference=DATA_REFERENCE_POINT,  # TODO update
             description=(
                 "Max- and mean-detected channel power vs. time, with "
                 + f"an integration time of {p[TD_BIN_SIZE_MS]} ms. "
@@ -1066,7 +1065,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pfp_x_axis__s[-1]],
             x_step=[pfp_x_axis__s[1] - pfp_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,
+            reference=DATA_REFERENCE_POINT,  # TODO update
             description=(
                 "Channelized periodic frame power statistics reported over"
                 + f" a {p[PFP_FRAME_PERIOD_MS]} ms frame period, with frame resolution"
@@ -1122,11 +1121,13 @@ def create_capture_segment(
             duration=measurement_result[DURATION_MS],
             overload=measurement_result["overload"],
             sensor_calibration=ntia_sensor.Calibration(
-                datetime=measurement_result["sensor_cal"]["datetime"],
-                gain=round(measurement_result["sensor_cal"]["gain"], 3),
-                noise_figure=round(measurement_result["sensor_cal"]["noise_figure"], 3),
+                datetime=self.sensor.sensor_calibration_data["datetime"],
+                gain=round(measurement_result["applied_calibration"]["gain"], 3),
+                noise_figure=round(
+                    measurement_result["applied_calibration"]["noise_figure"], 3
+                ),
                 temperature=round(measurement_result["sensor_cal"]["temperature"], 1),
-                reference=DATA_REFERENCE_POINT,
+                reference=measurement_result["reference"],
             ),
             sigan_settings=ntia_sensor.SiganSettings(
                 reference_level=self.sensor.signal_analyzer.reference_level,
diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index aa6f16ba..a54cb785 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -152,10 +152,6 @@ def __init__(self, parameters: dict):
         self.classification = get_parameter(CLASSIFICATION, self.parameters)
         self.cal_adjust = get_parameter(CAL_ADJUST, self.parameters)
         assert isinstance(self.cal_adjust, bool)
-        if self.cal_adjust:
-            self.data_reference = "calibration terminal"
-        else:
-            self.data_reference = "signal analyzer input"
         # FFT setup
         self.fft_detector = create_statistical_detector(
             "M4sDetector", ["min", "max", "mean", "median", "sample"]
@@ -179,7 +175,7 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result.update(self.parameters)
         measurement_result[
             "calibration_datetime"
-        ] = self.sensor.signal_analyzer.sensor_calibration_data["datetime"]
+        ] = self.sensor.sensor_calibration_data["datetime"]
         measurement_result["task_id"] = task_id
         measurement_result["classification"] = self.classification
 
@@ -269,7 +265,7 @@ def create_metadata(self, measurement_result: dict, recording: int = None) -> No
             x_stop=[frequencies[-1]],
             x_step=[frequencies[1] - frequencies[0]],
             y_units="dBm",
-            reference=self.data_reference,
+            reference=measurement_result["reference"],
             description=(
                 "Results of min, max, mean, and median statistical detectors, "
                 + f"along with a random sampling, from a set of {self.nffts} "
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 0eb55655..a2261fbf 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -91,7 +91,7 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result["task_id"] = task_id
         measurement_result[
             "calibration_datetime"
-        ] = self.sensor.signal_analyzer.sensor_calibration_data["datetime"]
+        ] = self.sensor.sensor_calibration_data["datetime"]
         measurement_result["classification"] = self.classification
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index bd114e57..72995f14 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -117,15 +117,18 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 overload=measurement_result["overload"],
                 sigan_settings=sigan_settings,
             )
-            sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
+            sensor_cal = self.sensor.sensor_calibration_data
             if sensor_cal is not None:
                 if "1db_compression_point" in sensor_cal:
                     sensor_cal["compression_point"] = sensor_cal.pop(
                         "1db_compression_point"
                     )
-                    capture_segment.sensor_calibration = ntia_sensor.Calibration(
-                        **sensor_cal
-                    )
+                if "reference" not in sensor_cal:
+                    # If the calibration data already includes this, don't overwrite
+                    sensor_cal["reference"] = measurement_result["reference"]
+                capture_segment.sensor_calibration = ntia_sensor.Calibration(
+                    **sensor_cal
+                )
             measurement_result["capture_segment"] = capture_segment
 
             self.create_metadata(measurement_result, recording_id)
diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index d26bb743..bf1099e7 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -228,10 +228,8 @@ def calibrate(self, params: dict):
 
         # Get noise diode on IQ
         logger.debug("Acquiring IQ samples with noise diode ON")
-        noise_on_measurement_result = (
-            self.sensor.signal_analyzer.acquire_time_domain_samples(
-                num_samples, num_samples_skip=nskip, cal_adjust=False
-            )
+        noise_on_measurement_result = self.sensor.acquire_time_domain_samples(
+            num_samples, nskip, cal_adjust=False
         )
         sample_rate = noise_on_measurement_result["sample_rate"]
 
@@ -242,10 +240,8 @@ def calibrate(self, params: dict):
 
         # Get noise diode off IQ
         logger.debug("Acquiring IQ samples with noise diode OFF")
-        noise_off_measurement_result = (
-            self.sensor.signal_analyzer.acquire_time_domain_samples(
-                num_samples, num_samples_skip=nskip, cal_adjust=False
-            )
+        noise_off_measurement_result = self.sensor.acquire_time_domain_samples(
+            num_samples, nskip, cal_adjust=False
         )
         assert (
             sample_rate == noise_off_measurement_result["sample_rate"]
@@ -259,24 +255,22 @@ def calibrate(self, params: dict):
             noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"])
             noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"])
         else:
-            if self.sensor.signal_analyzer.sensor_calibration.is_default:
+            if self.sensor.sensor_calibration.is_default:
                 raise Exception(
                     "Calibrations without IIR filter cannot be performed with default calibration."
                 )
 
             logger.debug("Skipping IIR filtering")
             # Get ENBW from sensor calibration
-            assert set(
-                self.sensor.signal_analyzer.sensor_calibration.calibration_parameters
-            ) <= set(
+            assert set(self.sensor.sensor_calibration.calibration_parameters) <= set(
                 sigan_params.keys()
             ), f"Action parameters do not include all required calibration parameters"
             cal_args = [
                 sigan_params[k]
-                for k in self.sensor.signal_analyzer.sensor_calibration.calibration_parameters
+                for k in self.sensor.sensor_calibration.calibration_parameters
             ]
-            self.sensor.signal_analyzer.recompute_sensor_calibration_data(cal_args)
-            enbw_hz = self.sensor.signal_analyzer.sensor_calibration_data["enbw"]
+            self.sensor.recompute_sensor_calibration_data(cal_args)
+            enbw_hz = self.sensor.sensor_calibration_data["enbw"]
             noise_on_data = noise_on_measurement_result["data"]
             noise_off_data = noise_off_measurement_result["data"]
 
@@ -294,7 +288,7 @@ def calibrate(self, params: dict):
         )
 
         # Update sensor calibration with results
-        self.sensor.signal_analyzer.sensor_calibration.update(
+        self.sensor.sensor_calibration.update(
             sigan_params, utils.get_datetime_str_now(), gain, noise_figure, temp_c
         )
 
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index d3299ca4..8d16d763 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -51,7 +51,7 @@ def create_capture_segment(
             overload=overload,
             sigan_settings=sigan_settings,
         )
-        sensor_cal = self.sensor.signal_analyzer.sensor_calibration_data
+        sensor_cal = self.sensor.sensor_calibration_data
         # Rename compression point keys if they exist
         # then set calibration metadata if it exists
         if sensor_cal is not None:
@@ -161,7 +161,7 @@ def acquire_data(
             + f" and {'' if cal_adjust else 'not '}applying gain adjustment based"
             + " on calibration data"
         )
-        measurement_result = self.sensor.signal_analyzer.acquire_time_domain_samples(
+        measurement_result = self.sensor.acquire_time_domain_samples(
             num_samples,
             num_samples_skip=nskip,
             cal_adjust=cal_adjust,
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 91f9a273..cbd3e351 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -239,11 +239,10 @@ def acquire_time_domain_samples(
         if cal_adjust:
             if self.sensor_calibration is not None:
                 logger.debug("Scaling samples using calibration data")
-                calibrated_gain__db = 0.0
                 self.recompute_sensor_calibration_data()
-                sensor_gain = self.sensor_calibration_data["gain"]
-                logger.debug(f"Using sensor gain: {sensor_gain} dB")
-                calibrated_gain__db += sensor_gain
+                calibrated_gain__db = self.sensor_calibration_data["gain"]
+                calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
+                logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     # TODO recompute functions match to current signal analyzer
@@ -252,6 +251,7 @@ def acquire_time_domain_samples(
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss
+                    calibrated_nf__db += differential_loss
                     measurement_result[
                         "reference"
                     ] = self.differential_calibration.reference_point
@@ -259,9 +259,17 @@ def acquire_time_domain_samples(
                     # No differential calibration exists
                     logger.debug("No differential calibration was applied")
                     measurement_result["reference"] = "calibration terminal"
+
                 linear_gain = 10.0 ** (calibrated_gain__db / 20.0)
-                logger.debug(f"Applying gain of {linear_gain}")
+                logger.debug(f"Applying total gain of {calibrated_gain__db}")
                 measurement_result["data"] /= linear_gain
+
+                # Metadata: record the gain and noise figure based on the actual
+                # scaling which was used.
+                measurement_result["applied_calibration"] = {
+                    "gain": calibrated_gain__db,
+                    "noise_figure": calibrated_nf__db,
+                }
             else:
                 # No sensor calibration exists
                 msg = "Unable to scale samples without sensor calibration data"
@@ -270,5 +278,6 @@ def acquire_time_domain_samples(
         else:
             # Set the data reference in the measurement_result
             measurement_result["reference"] = "signal analyzer input"
+            measurement_result["calibration"] = None
 
         return measurement_result

From eba790a42c81ecf6cd075c7a8cd7b17c3938744d Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 24 Jan 2024 13:33:34 -0500
Subject: [PATCH 20/62] generalize differential calibration module docstring

---
 .../calibration/differential_calibration.py    | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
index 4cf40322..e25ad71d 100644
--- a/scos_actions/calibration/differential_calibration.py
+++ b/scos_actions/calibration/differential_calibration.py
@@ -1,20 +1,20 @@
 """
 Dataclass implementation for "differential calibration" handling.
 
-A differential calibration provides loss values at different frequencies
-which represent excess loss between the calibration terminal and the antenna
-port. At present, this is measured manually using a calibration probe consisting
-of a calibrated noise source and a programmable attenuator.
+A differential calibration provides loss values which represent excess loss
+between the `SensorCalibration` reference point and another reference point.
+A typical usage would be for calibrating out measured cable losses which exist
+between the antenna and the Y-factor calibration terminal. At present, this is
+measured manually using a calibration probe consisting of a calibrated noise
+source and a programmable attenuator.
 
 The ``reference_point`` top-level key defines the point to which measurements
 are referenced after using the correction factors included in the file.
 
 The ``calibration_data`` entries are expected to include these correction factors,
-with the key name ``"loss"`` and values in decibels (dB). These correction factors
-represent the differential loss between the calibration terminal used by onboard
-``SensorCalibration`` results and the reference point defined by ``reference_point``.
-A positive value of ``"loss"`` indicates a LOSS going FROM ``reference_point`` TO
-the calibration terminal used by the ``SensorCalibration``.
+with the key name ``"loss"`` and values in decibels (dB). A positive value of
+``"loss"`` indicates a LOSS going FROM ``reference_point`` TO the calibration
+terminal used by the ``SensorCalibration``.
 """
 
 from dataclasses import dataclass

From fdea1a7aa875560468a69fcc8b444128461c4f51 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 24 Jan 2024 18:42:02 -0500
Subject: [PATCH 21/62] make calibration data a sensor property

automatically recompute calibration data in the getter method
---
 scos_actions/hardware/sensor.py | 61 +++++++++++++++++++++------------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index cbd3e351..6143b0a5 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -2,7 +2,7 @@
 import hashlib
 import json
 import logging
-from typing import Dict, Optional
+from typing import Any, Dict, Optional
 
 from its_preselector.preselector import Preselector
 from its_preselector.web_relay import WebRelay
@@ -37,8 +37,9 @@ def __init__(
         self.switches = switches
         self.location = location
         self.capabilities = capabilities
-        self.sensor_calibration_data = {}
+        self._sensor_calibration_data = {}
         self._sensor_calibration = sensor_cal
+        self._differential_calibration_data = {}
         self._differential_calibration = differential_cal
         # There is no setter for start_time property
         self._start_time = datetime.datetime.utcnow()
@@ -134,47 +135,67 @@ def differential_calibration(self, cal: Optional[DifferentialCalibration]):
 
     @property
     def last_calibration_time(self) -> str:
-        """Get a datetime string for the most recent sensor calibration."""
+        """A datetime string for the most recent sensor calibration."""
         return convert_string_to_millisecond_iso_format(
             self.sensor_calibration.last_calibration_datetime
         )
 
+    @property
+    def sensor_calibration_data(self) -> Dict[str, Any]:
+        """Sensor calibration data for the current sensor settings."""
+        self._recompute_sensor_calibration_data()
+        return self._sensor_calibration_data
+
+    @property
+    def differential_calibration_data(self) -> Dict[str, float]:
+        """Differential calibration data for the current sensor settings."""
+        self._recompute_differential_calibration_data()
+        return self._differential_calibration_data
+
     def _get_calibration_args_from_sigan(self, calibration: Calibration) -> list:
         """Get current values of signal analyzer settings which are calibration parameters."""
-        cal_params = [
-            p for p in calibration.calibration_parameters if p in SIGAN_SETTINGS_KEYS
-        ]
-        if set(cal_params) <= set(calibration.calibration_parameters):
+        try:
+            # Get calibration parameters which are valid settings for signal analyzers
+            cal_params = [
+                p
+                for p in calibration.calibration_parameters
+                if p in SIGAN_SETTINGS_KEYS
+            ]
+            if len(cal_params) == 0:
+                err_text = "any"  # Formats error message below
+                raise ValueError
+            else:
+                err_text = "this"  # Formats error message below
+            cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
+        except Exception as e:
             msg = (
-                "One or more required calibration parameters is not a valid signal "
-                + f"analyzer property.\nRequired parameters: {calibration.calibration_parameters}"
-                + f"\nSignal analyzer properties: {list(vars(self.signal_analyzer).keys())}"
+                f"One or more calibration parameters is not a valid setting for {err_text} "
+                + f"signal analyzer.\nRequired parameters: {calibration.calibration_parameters}"
             )
-            logger.error(msg)
-            raise KeyError
-        cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
+            logger.exception(msg)
+            raise e
         logger.debug(f"Matched calibration params: {cal_args}")
         return cal_args  # Order matches calibration.calibration_parameters
 
-    def recompute_differential_calibration_data(self) -> None:
+    def _recompute_differential_calibration_data(self) -> None:
         """Set the differential calibration data based on the current tuning."""
-        self.differential_calibration_data = {}
+        self._differential_calibration_data = {}
         if self.differential_calibration is not None:
             cal_args = self._get_calibration_args_from_sigan(
                 self.differential_calibration
             )
-            self.differential_calibration_data.update(
+            self._differential_calibration_data.update(
                 self.differential_calibration.get_calibration_dict(cal_args)
             )
         else:
             logger.warning("Differential calibration does not exist.")
 
-    def recompute_sensor_calibration_data(self) -> None:
+    def _recompute_sensor_calibration_data(self) -> None:
         """Set the sensor calibration data based on the current tuning."""
-        self.sensor_calibration_data = {}
+        self._sensor_calibration_data = {}
         if self.sensor_calibration is not None:
             cal_args = self._get_calibration_args_from_sigan(self.sensor_calibration)
-            self.sensor_calibration_data.update(
+            self._sensor_calibration_data.update(
                 self.sensor_calibration.get_calibration_dict(cal_args)
             )
         else:
@@ -239,7 +260,6 @@ def acquire_time_domain_samples(
         if cal_adjust:
             if self.sensor_calibration is not None:
                 logger.debug("Scaling samples using calibration data")
-                self.recompute_sensor_calibration_data()
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
@@ -247,7 +267,6 @@ def acquire_time_domain_samples(
                     # Also apply differential calibration correction
                     # TODO recompute functions match to current signal analyzer
                     # settings. should they use the measurement_result instead?
-                    self.recompute_differential_calibration_data()
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss

From 3dfdb522182d4496564a022555b729395ab93ad2 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 10:14:29 -0500
Subject: [PATCH 22/62] implement mock sensor for testing

---
 .../tests/test_acquire_single_freq_fft.py     |  8 +--
 .../actions/tests/test_monitor_sigan.py       | 10 ++--
 .../tests/test_single_freq_tdomain_iq.py      | 10 ++--
 .../tests/test_stepped_freq_tdomain_iq.py     |  9 +--
 scos_actions/actions/tests/test_sync_gps.py   |  6 +-
 scos_actions/hardware/mocks/mock_sensor.py    | 59 +++++++++++++++++++
 scos_actions/hardware/mocks/mock_sigan.py     | 14 -----
 scos_actions/hardware/tests/test_sensor.py    | 26 ++++++--
 scos_actions/hardware/tests/test_sigan.py     | 12 ++--
 9 files changed, 105 insertions(+), 49 deletions(-)
 create mode 100644 scos_actions/hardware/mocks/mock_sensor.py

diff --git a/scos_actions/actions/tests/test_acquire_single_freq_fft.py b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
index a172f36f..8a4bdf3e 100644
--- a/scos_actions/actions/tests/test_acquire_single_freq_fft.py
+++ b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
@@ -1,7 +1,6 @@
 from scos_actions.actions.tests.utils import check_metadata_fields
 from scos_actions.discover import test_actions as actions
-from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.signals import measurement_action_completed
 
 SINGLE_FREQUENCY_FFT_ACQUISITION = {
@@ -29,9 +28,8 @@ def callback(sender, **kwargs):
     measurement_action_completed.connect(callback)
     action = actions["test_single_frequency_m4s_action"]
     assert action.description
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer())
     action(
-        sensor=sensor,
+        sensor=MockSensor(),
         schedule_entry=SINGLE_FREQUENCY_FFT_ACQUISITION,
         task_id=1,
     )
@@ -89,6 +87,6 @@ def callback(sender, **kwargs):
 def test_num_samples_skip():
     action = actions["test_single_frequency_m4s_action"]
     assert action.description
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer())
+    sensor = MockSensor()
     action(sensor, SINGLE_FREQUENCY_FFT_ACQUISITION, 1)
     assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_monitor_sigan.py b/scos_actions/actions/tests/test_monitor_sigan.py
index 0767ea84..38085efd 100644
--- a/scos_actions/actions/tests/test_monitor_sigan.py
+++ b/scos_actions/actions/tests/test_monitor_sigan.py
@@ -1,6 +1,6 @@
 from scos_actions.discover import test_actions as actions
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
 from scos_actions.signals import trigger_api_restart
 
 MONITOR_SIGAN_SCHEDULE = {
@@ -23,10 +23,10 @@ def callback(sender, **kwargs):
     action = actions["test_monitor_sigan"]
     mock_sigan = MockSignalAnalyzer()
     mock_sigan._is_available = False
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     action(sensor, MONITOR_SIGAN_SCHEDULE, 1)
     assert _api_restart_triggered == True  # signal sent
-    mock_sigan._is_available = True
+    sensor.signal_analyzer._is_available = True
 
 
 def test_monitor_sigan_not_healthy():
@@ -40,7 +40,7 @@ def callback(sender, **kwargs):
     action = actions["test_monitor_sigan"]
     mock_sigan = MockSignalAnalyzer()
     mock_sigan.times_to_fail_recv = 6
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     action(sensor, MONITOR_SIGAN_SCHEDULE, 1)
     assert _api_restart_triggered == True  # signal sent
 
@@ -57,6 +57,6 @@ def callback(sender, **kwargs):
     mock_sigan = MockSignalAnalyzer()
     mock_sigan._is_available = True
     mock_sigan.set_times_to_fail_recv(0)
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     action(sensor, MONITOR_SIGAN_SCHEDULE, 1)
     assert _api_restart_triggered == False  # signal not sent
diff --git a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
index fa5a1ad4..c5d4eacd 100644
--- a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
@@ -1,9 +1,8 @@
 import pytest
-
 from scos_actions.actions.tests.utils import check_metadata_fields
 from scos_actions.discover import test_actions as actions
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
 from scos_actions.signals import measurement_action_completed
 
 SINGLE_TIMEDOMAIN_IQ_ACQUISITION = {
@@ -31,7 +30,7 @@ def callback(sender, **kwargs):
     measurement_action_completed.connect(callback)
     action = actions["test_single_frequency_iq_action"]
     assert action.description
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer())
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     assert _data.any()
     assert _metadata
@@ -62,7 +61,7 @@ def test_required_components():
     action = actions["test_single_frequency_m4s_action"]
     mock_sigan = MockSignalAnalyzer()
     mock_sigan._is_available = False
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor(signal_analyzer=mock_sigan)
     with pytest.raises(RuntimeError):
         action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     mock_sigan._is_available = True
@@ -71,7 +70,6 @@ def test_required_components():
 def test_num_samples_skip():
     action = actions["test_single_frequency_iq_action"]
     assert action.description
-    mock_sigan = MockSignalAnalyzer()
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
index 40c44141..d389e024 100644
--- a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
@@ -1,6 +1,5 @@
 from scos_actions.discover import test_actions as actions
-from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.sensor import Sensor
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.signals import measurement_action_completed
 
 SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION = {
@@ -34,8 +33,7 @@ def callback(sender, **kwargs):
     measurement_action_completed.connect(callback)
     action = actions["test_multi_frequency_iq_action"]
     assert action.description
-    mock_sigan = MockSignalAnalyzer()
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION, 1)
     for i in range(_count):
         assert _datas[i].any()
@@ -48,8 +46,7 @@ def callback(sender, **kwargs):
 def test_num_samples_skip():
     action = actions["test_multi_frequency_iq_action"]
     assert action.description
-    mock_sigan = MockSignalAnalyzer()
-    sensor = Sensor(signal_analyzer=mock_sigan)
+    sensor = MockSensor()
     action(sensor, SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION, 1)
     if isinstance(action.parameters["nskip"], list):
         assert (
diff --git a/scos_actions/actions/tests/test_sync_gps.py b/scos_actions/actions/tests/test_sync_gps.py
index 5b8a9311..694c4647 100644
--- a/scos_actions/actions/tests/test_sync_gps.py
+++ b/scos_actions/actions/tests/test_sync_gps.py
@@ -2,10 +2,8 @@
 import sys
 
 import pytest
-
 from scos_actions.discover import test_actions
-from scos_actions.hardware.mocks.mock_gps import MockGPS
-from scos_actions.hardware.sensor import Sensor
+from scos_actions.hardware.mocks.mock_sensor import MockSensor
 from scos_actions.signals import location_action_completed
 
 SYNC_GPS = {
@@ -29,7 +27,7 @@ def callback(sender, **kwargs):
 
     location_action_completed.connect(callback)
     action = test_actions["test_sync_gps"]
-    sensor = Sensor(gps=MockGPS())
+    sensor = MockSensor()
     if sys.platform == "linux":
         action(sensor, SYNC_GPS, 1)
         assert _latitude
diff --git a/scos_actions/hardware/mocks/mock_sensor.py b/scos_actions/hardware/mocks/mock_sensor.py
new file mode 100644
index 00000000..4e432f9f
--- /dev/null
+++ b/scos_actions/hardware/mocks/mock_sensor.py
@@ -0,0 +1,59 @@
+import logging
+
+from scos_actions.hardware.mocks.mock_gps import MockGPS
+from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
+from scos_actions.hardware.sensor import Sensor
+from scos_actions.utils import get_datetime_str_now
+
+_mock_sensor_cal_data = {
+    "datetime": get_datetime_str_now(),
+    "gain": 0,
+    "enbw": None,
+    "noise_figure": None,
+    "1db_compression_point": None,
+    "temperature": 26.85,
+}
+
+_mock_differential_cal_data = {"loss": 10.0}
+
+_mock_capabilities = {"sensor": {}}
+
+_mock_location = {"x": -999, "y": -999, "z": -999, "description": "Testing"}
+
+logger = logging.getLogger(__name__)
+
+
+class MockSensor(Sensor):
+    def __init__(
+        self,
+        signal_analyzer=MockSignalAnalyzer(),
+        gps=MockGPS(),
+        preselector=None,
+        switches={},
+        location=_mock_location,
+        capabilities=_mock_capabilities,
+        sensor_cal=None,
+        differential_cal=None,
+    ):
+        if (sensor_cal is not None) or (differential_cal is not None):
+            logger.warning(
+                "Calibration object provided to mock sensor will not be used to query calibration data."
+            )
+        super().__init__(
+            signal_analyzer,
+            gps,
+            preselector,
+            switches,
+            location,
+            capabilities,
+            sensor_cal,
+            differential_cal,
+        )
+
+    @property
+    def sensor_calibration_data(self):
+        return _mock_sensor_cal_data
+
+    @property
+    def differential_calibration_data(self):
+        return _mock_differential_cal_data
diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 03a9e42b..00a31804 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -31,15 +31,6 @@ def __init__(
         randomize_values: bool = False,
     ):
         super().__init__(switches)
-        # Define the default calibration dict
-        self.DEFAULT_SENSOR_CALIBRATION = {
-            "datetime": get_datetime_str_now(),
-            "gain": 0,  # Defaults to sigan gain
-            "enbw": None,  # Defaults to sigan enbw
-            "noise_figure": None,  # Defaults to sigan noise figure
-            "1db_compression_point": None,  # Defaults to sigan compression + preselector gain
-            "temperature": 26.85,
-        }
         self.auto_dc_offset = False
         self._frequency = 700e6
         self._sample_rate = 10e6
@@ -61,7 +52,6 @@ def __init__(
         self.times_failed_recv = 0
 
         self.randomize_values = randomize_values
-        self.sensor_calibration_data = self.DEFAULT_SENSOR_CALIBRATION
 
     @property
     def is_available(self):
@@ -188,7 +178,3 @@ def acquire_time_domain_samples(
     def set_times_to_fail_recv(self, n):
         self.times_to_fail_recv = n
         self.times_failed_recv = 0
-
-    @property
-    def last_calibration_time(self):
-        return get_datetime_str_now()
diff --git a/scos_actions/hardware/tests/test_sensor.py b/scos_actions/hardware/tests/test_sensor.py
index 1d86de81..25e66ebb 100644
--- a/scos_actions/hardware/tests/test_sensor.py
+++ b/scos_actions/hardware/tests/test_sensor.py
@@ -1,10 +1,26 @@
-from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
-from scos_actions.hardware.mocks.mock_gps import MockGPS
-from scos_actions.hardware.sensor import Sensor
+import datetime
 
+from scos_actions.hardware.mocks.mock_sensor import (
+    MockSensor,
+    _mock_capabilities,
+    _mock_differential_cal_data,
+    _mock_location,
+    _mock_sensor_cal_data,
+)
 
-def test_sensor():
-    sensor = Sensor(signal_analyzer=MockSignalAnalyzer(), gps=MockGPS())
+
+def test_mock_sensor():
+    sensor = MockSensor()
     assert sensor is not None
     assert sensor.signal_analyzer is not None
     assert sensor.gps is not None
+    assert sensor.preselector is None
+    assert sensor.switches == {}
+    assert sensor.location == _mock_location
+    assert sensor.capabilities == _mock_capabilities
+    assert sensor.sensor_calibration is None
+    assert sensor.differential_calibration is None
+    assert sensor.has_configurable_preselector is False
+    assert sensor.sensor_calibration_data == _mock_sensor_cal_data
+    assert sensor.differential_calibration_data == _mock_differential_cal_data
+    assert isinstance(sensor.start_time, datetime.datetime)
diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py
index c82ffeec..53081aa0 100644
--- a/scos_actions/hardware/tests/test_sigan.py
+++ b/scos_actions/hardware/tests/test_sigan.py
@@ -1,8 +1,12 @@
+import pytest
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
 
 
-def test_sigan_default_cal():
+def test_mock_sigan():
     sigan = MockSignalAnalyzer()
-    sigan.recompute_sensor_calibration_data([])
-    sensor_cal = sigan.sensor_calibration_data
-    assert sensor_cal["gain"] == 0
+    # Test default values are available as properties
+    assert sigan.frequency == sigan._frequency
+    assert sigan.sample_rate == sigan._sample_rate
+    assert sigan.gain == sigan._gain
+    assert sigan.attenuation == sigan._attenuation
+    assert sigan.preamp_enable == sigan._preamp_enable

From 89e5ae7df54ad5291fe3195a85fe553e43c85159 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 10:38:43 -0500
Subject: [PATCH 23/62] simplify and update mock sigan

- Remove outdated and unnecessary instance variables
- Remove retry logic from acquire_samples to account for changes in the Sensor object
- Remove redundant properties that are the same as the base class
---
 scos_actions/hardware/mocks/mock_sigan.py | 90 +++++++++--------------
 1 file changed, 35 insertions(+), 55 deletions(-)

diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index 00a31804..4b678a6e 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -31,7 +31,7 @@ def __init__(
         randomize_values: bool = False,
     ):
         super().__init__(switches)
-        self.auto_dc_offset = False
+        self._model = "Mock Signal Analyzer"
         self._frequency = 700e6
         self._sample_rate = 10e6
         self.clock_rate = 40e6
@@ -39,8 +39,6 @@ def __init__(
         self._attenuation = 0
         self._preamp_enable = False
         self._reference_level = -30
-        self._overload = False
-        self._capture_time = None
         self._is_available = True
         self._plugin_version = SCOS_ACTIONS_VERSION
         self._firmware_version = "1.2.3"
@@ -61,14 +59,6 @@ def is_available(self):
     def plugin_version(self):
         return self._plugin_version
 
-    @property
-    def firmware_version(self):
-        return self._firmware_version
-
-    @property
-    def api_version(self):
-        return self._api_version
-
     @property
     def sample_rate(self):
         return self._sample_rate
@@ -124,56 +114,46 @@ def connect(self):
         pass
 
     def acquire_time_domain_samples(
-        self, num_samples: int, num_samples_skip: int = 0, retries: int = 5
+        self, num_samples: int, num_samples_skip: int = 0
     ) -> dict:
         logger.warning("Using mock signal analyzer!")
-        self.sigan_overload = False
-        self._capture_time = None
-        self._num_samples_skip = num_samples_skip
+        overload = False
+        capture_time = None
 
         # Try to acquire the samples
-        max_retries = retries
         data = []
-        while True:
-            if self.times_failed_recv < self.times_to_fail_recv:
-                self.times_failed_recv += 1
-                data = np.ones(0, dtype=np.complex64)
-            else:
-                self._capture_time = get_datetime_str_now()
-                if self.randomize_values:
-                    i = np.random.normal(0.5, 0.5, num_samples)
-                    q = np.random.normal(0.5, 0.5, num_samples)
-                    rand_iq = np.empty(num_samples, dtype=np.complex64)
-                    rand_iq.real = i
-                    rand_iq.imag = q
-                    data = rand_iq
-                else:
-                    data = np.ones(num_samples, dtype=np.complex64)
-
-            data_len = len(data)
-            if not len(data) == num_samples:
-                if retries > 0:
-                    msg = "Signal analyzer error: requested {} samples, but got {}."
-                    logger.warning(msg.format(num_samples + num_samples_skip, data_len))
-                    logger.warning(f"Retrying {retries} more times.")
-                    retries = retries - 1
-                else:
-                    err = "Failed to acquire correct number of samples "
-                    err += f"{max_retries} times in a row."
-                    raise RuntimeError(err)
+        if self.times_failed_recv < self.times_to_fail_recv:
+            self.times_failed_recv += 1
+            data = np.ones(0, dtype=np.complex64)
+        else:
+            capture_time = get_datetime_str_now()
+            if self.randomize_values:
+                i = np.random.normal(0.5, 0.5, num_samples)
+                q = np.random.normal(0.5, 0.5, num_samples)
+                rand_iq = np.empty(num_samples, dtype=np.complex64)
+                rand_iq.real = i
+                rand_iq.imag = q
+                data = rand_iq
             else:
-                logger.debug(f"Successfully acquired {num_samples} samples.")
-                return {
-                    "data": data,
-                    "overload": self._overload,
-                    "frequency": self._frequency,
-                    "gain": self._gain,
-                    "attenuation": self._attenuation,
-                    "preamp_enable": self._preamp_enable,
-                    "reference_level": self._reference_level,
-                    "sample_rate": self._sample_rate,
-                    "capture_time": self._capture_time,
-                }
+                data = np.ones(num_samples, dtype=np.complex64)
+
+        if (data_len := len(data)) != num_samples:
+            err = "Failed to acquire correct number of samples: "
+            err += f"got {data_len} instead of {num_samples}"
+            raise RuntimeError(err)
+        else:
+            logger.debug(f"Successfully acquired {num_samples} samples.")
+            return {
+                "data": data,
+                "overload": overload,
+                "frequency": self._frequency,
+                "gain": self._gain,
+                "attenuation": self._attenuation,
+                "preamp_enable": self._preamp_enable,
+                "reference_level": self._reference_level,
+                "sample_rate": self._sample_rate,
+                "capture_time": capture_time,
+            }
 
     def set_times_to_fail_recv(self, n):
         self.times_to_fail_recv = n

From 59b87cacdb6cd94529fe473590122d1d483278c6 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 10:39:41 -0500
Subject: [PATCH 24/62] Make firmware and API version properties usable

These properties no longer need to be overriden. Subclass constructors can just set the correct instance variables instead.
---
 scos_actions/hardware/sigan_iface.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index f7fb0bed..cb0e85cb 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -26,6 +26,8 @@ def __init__(
         switches: Optional[Dict[str, WebRelay]] = None,
     ):
         self._model = "Unknown"
+        self._api_version = "Unknown"
+        self._firmware_version = "Unknown"
         self.switches = switches
 
     @property
@@ -43,12 +45,12 @@ def plugin_version(self) -> str:
     @property
     def firmware_version(self) -> str:
         """Returns the version of the signal analyzer firmware."""
-        return "Unknown"
+        return self._firmware_version
 
     @property
     def api_version(self) -> str:
         """Returns the version of the underlying signal analyzer API."""
-        return "Unknown"
+        return self._api_version
 
     @abstractmethod
     def acquire_time_domain_samples(

From e75619561db495ab178ce14e9ce2d8aa32d0c03e Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 12:06:12 -0500
Subject: [PATCH 25/62] Update test_sigan.py

---
 scos_actions/hardware/tests/test_sigan.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py
index 53081aa0..d4434d78 100644
--- a/scos_actions/hardware/tests/test_sigan.py
+++ b/scos_actions/hardware/tests/test_sigan.py
@@ -5,8 +5,14 @@
 def test_mock_sigan():
     sigan = MockSignalAnalyzer()
     # Test default values are available as properties
+    assert sigan.model == sigan._model
     assert sigan.frequency == sigan._frequency
     assert sigan.sample_rate == sigan._sample_rate
     assert sigan.gain == sigan._gain
     assert sigan.attenuation == sigan._attenuation
     assert sigan.preamp_enable == sigan._preamp_enable
+    assert sigan.reference_level == sigan._reference_level
+    assert sigan.is_available == sigan._is_available
+    assert sigan.plugin_version == sigan._plugin_version
+    assert sigan.firmware_version == sigan._firmware_version
+    assert sigan.api_version == sigan._api_version

From f1b1a633e0bf8eaa754e940efd5a15fd89d80b5c Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 18:37:35 -0500
Subject: [PATCH 26/62] generalize matching of calibration params

- simplify recompute_calibration methods
- cal_params is now an argument of acquire_time_domain_samples, and is intended to be used by passing action.parameters dicts which contain the required key/values to match calibration data to the measurement
- add some specific calibration-related exceptions
- update actions and unit tests
---
 .../actions/acquire_sea_data_product.py       | 12 +--
 .../actions/acquire_single_freq_fft.py        |  2 +-
 .../actions/acquire_single_freq_tdomain_iq.py |  4 +-
 .../acquire_stepped_freq_tdomain_iq.py        |  4 +-
 .../actions/interfaces/measurement_action.py  |  7 +-
 .../tests/test_acquire_single_freq_fft.py     |  8 --
 .../tests/test_single_freq_tdomain_iq.py      |  8 --
 .../tests/test_stepped_freq_tdomain_iq.py     | 17 ----
 .../calibration/interfaces/calibration.py     | 73 +++++-------------
 .../calibration/sensor_calibration.py         | 22 +++++-
 .../calibration/tests/test_calibration.py     | 25 +-----
 .../tests/test_sensor_calibration.py          | 12 ++-
 scos_actions/calibration/utils.py             | 25 +++++-
 scos_actions/hardware/sensor.py               | 77 +++++++------------
 14 files changed, 118 insertions(+), 178 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 003db3b8..fcadb459 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -626,13 +626,11 @@ def capture_iq(self, params: dict) -> dict:
         nskip = utils.get_parameter(NUM_SKIP, params)
         num_samples = int(params[SAMPLE_RATE] * duration_ms * 1e-3)
         # Collect IQ data
-        measurement_result = self.sensor.acquire_time_domain_samples(num_samples, nskip)
+        measurement_result = self.sensor.acquire_time_domain_samples(
+            num_samples, nskip, cal_params=params
+        )
         # Store some metadata with the IQ
         measurement_result.update(params)
-        measurement_result["sensor_cal"] = self.sensor.sensor_calibration_data
-        measurement_result[
-            "differential_cal"
-        ] = self.sensor.differential_calibration_data
         toc = perf_counter()
         logger.debug(
             f"IQ Capture ({duration_ms} ms @ {(params[FREQUENCY]/1e6):.1f} MHz) completed in {toc-tic:.2f} s."
@@ -1126,7 +1124,9 @@ def create_capture_segment(
                 noise_figure=round(
                     measurement_result["applied_calibration"]["noise_figure"], 3
                 ),
-                temperature=round(measurement_result["sensor_cal"]["temperature"], 1),
+                temperature=round(
+                    self.sensor.sensor_calibration_data["temperature"], 1
+                ),
                 reference=measurement_result["reference"],
             ),
             sigan_settings=ntia_sensor.SiganSettings(
diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index a54cb785..21bc8f5e 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -164,7 +164,7 @@ def __init__(self, parameters: dict):
     def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Acquire IQ data and generate M4S result
         measurement_result = self.acquire_data(
-            self.num_samples, self.nskip, self.cal_adjust
+            self.num_samples, self.nskip, self.cal_adjust, cal_params=self.parameters
         )
         # Actual sample rate may differ from configured value
         sample_rate_Hz = measurement_result["sample_rate"]
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index a2261fbf..6ff6b537 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -84,7 +84,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Use the sigan's actual reported instead of requested sample rate
         sample_rate = self.sensor.signal_analyzer.sample_rate
         num_samples = int(sample_rate * self.duration_ms * 1e-3)
-        measurement_result = self.acquire_data(num_samples, self.nskip, self.cal_adjust)
+        measurement_result = self.acquire_data(
+            num_samples, self.nskip, self.cal_adjust, cal_params=self.parameters
+        )
         end_time = utils.get_datetime_str_now()
         measurement_result.update(self.parameters)
         measurement_result["end_time"] = end_time
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 72995f14..7278e5f1 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -100,7 +100,9 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             cal_adjust = get_parameter(CAL_ADJUST, measurement_params)
             sample_rate = self.sensor.signal_analyzer.sample_rate
             num_samples = int(sample_rate * duration_ms * 1e-3)
-            measurement_result = super().acquire_data(num_samples, nskip, cal_adjust)
+            measurement_result = super().acquire_data(
+                num_samples, nskip, cal_adjust, cal_params=measurement_params
+            )
             measurement_result.update(measurement_params)
             end_time = utils.get_datetime_str_now()
             measurement_result["end_time"] = end_time
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index 8d16d763..fe5e197c 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -154,7 +154,11 @@ def send_signals(self, task_id, metadata, measurement_data):
         )
 
     def acquire_data(
-        self, num_samples: int, nskip: int = 0, cal_adjust: bool = True
+        self,
+        num_samples: int,
+        nskip: int = 0,
+        cal_adjust: bool = True,
+        cal_params: Optional[dict] = None,
     ) -> dict:
         logger.debug(
             f"Acquiring {num_samples} IQ samples, skipping the first {nskip} samples"
@@ -165,6 +169,7 @@ def acquire_data(
             num_samples,
             num_samples_skip=nskip,
             cal_adjust=cal_adjust,
+            cal_params=cal_params,
         )
 
         return measurement_result
diff --git a/scos_actions/actions/tests/test_acquire_single_freq_fft.py b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
index 8a4bdf3e..e4d9c9fb 100644
--- a/scos_actions/actions/tests/test_acquire_single_freq_fft.py
+++ b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
@@ -82,11 +82,3 @@ def callback(sender, **kwargs):
             ]
         ]
     )
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_m4s_action"]
-    assert action.description
-    sensor = MockSensor()
-    action(sensor, SINGLE_FREQUENCY_FFT_ACQUISITION, 1)
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
index c5d4eacd..50c36b89 100644
--- a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
@@ -65,11 +65,3 @@ def test_required_components():
     with pytest.raises(RuntimeError):
         action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     mock_sigan._is_available = True
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_iq_action"]
-    assert action.description
-    sensor = MockSensor()
-    action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
index d389e024..2183d0bf 100644
--- a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
@@ -41,20 +41,3 @@ def callback(sender, **kwargs):
         assert _task_ids[i] == 1
         assert _recording_ids[i] == i + 1
     assert _count == 10
-
-
-def test_num_samples_skip():
-    action = actions["test_multi_frequency_iq_action"]
-    assert action.description
-    sensor = MockSensor()
-    action(sensor, SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION, 1)
-    if isinstance(action.parameters["nskip"], list):
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"][-1]
-        )
-    else:
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"]
-        )
diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 406e325e..42277253 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -3,9 +3,12 @@
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import Any, List, get_origin
+from typing import List, get_origin
 
-from scos_actions.calibration.utils import filter_by_parameter
+from scos_actions.calibration.utils import (
+    CalibrationParametersMissingException,
+    filter_by_parameter,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -38,65 +41,29 @@ def _validate_fields(self) -> None:
                     f"{c_name} field {f_name} must be {f_type}, not {actual_type}"
                 )
 
-    def get_calibration_dict(self, cal_params: List[Any]) -> dict:
+    def get_calibration_dict(self, params: dict) -> dict:
         """
-        Get calibration data closest to the specified parameter values.
-
-        :param cal_params: List of calibration parameter values. For example,
-            if ``calibration_parameters`` are ``["sample_rate", "gain"]``,
-            then the input to this method could be ``["15360000.0", "40"]``.
-        :return: The calibration data corresponding to the input parameter values.
-        """
-        cal_data = self.calibration_data
-        for i, setting_value in enumerate(cal_params):
-            setting = self.calibration_parameters[i]
-            logger.debug(f"Looking up calibration for {setting} at {setting_value}")
-            cal_data = filter_by_parameter(cal_data, setting_value)
-        logger.debug(f"Got calibration data: {cal_data}")
-
-        return cal_data
-
-    def _retrieve_data_to_update(self, params: dict) -> dict:
-        """
-        Locate the calibration data entry to update, based on a set
-        of calibration parameters.
+        Get calibration data entry at the specified parameter values.
 
         :param params: Parameters used for calibration. This must include
             entries for all of the ``Calibration.calibration_parameters``
             Example: ``{"sample_rate": 14000000.0, "attenuation": 10.0}``
-        :return: A dict containing the existing calibration entry at
-            the specified parameter set, which may be empty if none exists.
-        :raises ValueError: If not all calibration parameters exist as keys
-            in ``params``.
+        :return: The calibration data corresponding to the input parameter values.
         """
-        # Use params keys as calibration_parameters if none exist
-        if len(self.calibration_parameters) == 0:
-            logger.warning(
-                f"Setting required calibration parameters to {list(params.keys())}"
-            )
-            self.calibration_parameters = list(params.keys())
-        elif not set(params.keys()) >= set(self.calibration_parameters):
-            # Otherwise ensure all required parameters were used
-            raise ValueError(
-                "Not enough parameters specified to update calibration.\n"
-                + f"Required parameters are {self.calibration_parameters}"
+        # Check that input includes all required calibration parameters
+        if not set(params.keys()) >= set(self.calibration_parameters):
+            raise CalibrationParametersMissingException(
+                params, self.calibration_parameters
             )
+        cal_data = self.calibration_data
+        for p_name in self.calibration_parameters:
+            p_value = params[p_name]
+            logger.debug(f"Looking up calibration data at {p_name}={p_value}")
+            cal_data = filter_by_parameter(cal_data, p_value)
 
-        # Retrieve the existing calibration data entry based on
-        # the provided parameters and their values
-        data_entry = self.calibration_data
-        for parameter in self.calibration_parameters:
-            value = str(params[parameter]).lower()
-            logger.debug(f"Updating calibration at {parameter} = {value}")
-            try:
-                data_entry = data_entry[value]
-            except KeyError:
-                logger.debug(
-                    f"Creating required calibration data field for {parameter} = {value}"
-                )
-                data_entry[value] = {}
-                data_entry = data_entry[value]
-        return data_entry
+        logger.debug(f"Got calibration data: {cal_data}")
+
+        return cal_data
 
     @abstractmethod
     def update(self):
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 12997f22..de547fc9 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -3,6 +3,7 @@
 from typing import Dict, List, Union
 
 from scos_actions.calibration.interfaces.calibration import Calibration
+from scos_actions.calibration.utils import CalibrationEntryMissingException
 
 logger = logging.getLogger(__name__)
 
@@ -45,8 +46,25 @@ def update(
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
         :param file_path: File path for saving the updated calibration data.
         """
-        # Get existing calibration data entry which will be updated
-        data_entry = self._retrieve_data_to_update(params)
+        try:
+            # Get existing calibration data entry which will be updated
+            data_entry = self.get_calibration_dict(params)
+        except CalibrationEntryMissingException:
+            # Existing entry does not exist for these parameters. Make one.
+            data_entry = self.calibration_data
+            for p_name in self.calibration_parameters:
+                p_val = params[p_name]
+                try:
+                    data_entry = data_entry[p_val]
+                except KeyError:
+                    logger.debug(
+                        f"Creating calibration data field for {p_name}={p_val}"
+                    )
+                    data_entry[p_val] = {}
+                    data_entry = data_entry[p_val]
+        except Exception as e:
+            logger.exception("Failed to update calibration data.")
+            raise e
 
         # Update last calibration datetime
         self.last_calibration_datetime = calibration_datetime_str
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index a335d86f..9fb181e4 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -16,7 +16,7 @@ def setup_calibration_file(self, tmp_path: Path):
         """Create a dummy calibration file in the pytest temp directory."""
         # Create some dummy calibration data
         self.cal_params = ["frequency", "gain"]
-        self.frequencies = [3555e9, 3565e9, 3575e9]
+        self.frequencies = [3555e6, 3565e6, 3575e6]
         self.gains = [10.0, 20.0, 30.0]
         cal_data = {}
         for frequency in self.frequencies:
@@ -75,31 +75,10 @@ def test_field_validator(self):
     def test_get_calibration_dict(self):
         """Check the get_calibration_dict method with all dummy data."""
         for f in self.frequencies:
-            assert json.loads(
-                json.dumps(self.cal_data[f])
-            ) == self.sample_cal.get_calibration_dict([f])
             for g in self.gains:
                 assert json.loads(
                     json.dumps(self.cal_data[f][g])
-                ) == self.sample_cal.get_calibration_dict([f, g])
-
-    def test_retrieve_data_to_update(self):
-        """Check that the calibration data entry is correctly returned."""
-        for f in self.frequencies:
-            for g in self.gains:
-                params = {"frequency": f, "gain": g}
-                # Use the "is" keyword since this must not be a copy/identical dict
-                assert self.sample_cal.calibration_data[str(f)][
-                    str(g)
-                ] is self.sample_cal._retrieve_data_to_update(params)
-        # Method should work with len=0 calibration parameters
-        test_cal = Calibration([], {}, False, Path(""))
-        _ = test_cal._retrieve_data_to_update({"frequency": 3555e9, "gain": 10.0})
-        # And should fail if calibration parameters are not all supplied
-        with pytest.raises(ValueError):
-            _ = self.sample_cal._retrieve_data_to_update(
-                {"frequency": self.frequencies[0]}
-            )
+                ) == self.sample_cal.get_calibration_dict({"frequency": f, "gain": g})
 
     def test_to_and_from_json(self, tmp_path: Path):
         """Test the ``from_json`` factory method."""
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 6348c409..e9176718 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -61,7 +61,9 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
         calc_gain_sigan = easy_gain(sr_m, f_m, g_m)
 
         # Get the scale factor from the algorithm
-        interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+        interp_cal_data = self.sample_cal.get_calibration_dict(
+            {"sample_rate": sr, "frequency": f, "gain": g}
+        )
         interp_gain_siggan = interp_cal_data["gain"]
 
         # Save the point so we don't duplicate
@@ -91,7 +93,9 @@ def run_pytest_point(self, sr, f, g, reason, sr_m=False, f_m=False, g_m=False):
             )
         )
         if not isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance):
-            interp_cal_data = self.sample_cal.get_calibration_dict([sr, f, g])
+            interp_cal_data = self.sample_cal.get_calibration_dict(
+                {"sample_rate": sr, "frequency": f, "gain": g}
+            )
 
         assert isclose(calc_gain_sigan, interp_gain_siggan, abs_tol=tolerance), msg
         return True
@@ -247,7 +251,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
             clock_rate_lookup_by_sample_rate=[],
             sensor_uid="TESTING",
         )
-        cal_data = cal.get_calibration_dict([100.0, 200.0])
+        cal_data = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 200.0})
         assert cal_data["NF"] == "NF at 100, 200"
 
     def test_get_calibration_dict_within_range(self):
@@ -267,7 +271,7 @@ def test_get_calibration_dict_within_range(self):
             sensor_uid="TESTING",
         )
         with pytest.raises(CalibrationException) as e_info:
-            _ = cal.get_calibration_dict([100.0, 250.0])
+            _ = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 250.0})
         assert e_info.value.args[0] == (
             f"Could not locate calibration data at 250.0"
             + f"\nAttempted lookup using key '250.0'"
diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index b3aebdeb..6ddba00b 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -8,6 +8,25 @@ def __init__(self, msg):
         super().__init__(msg)
 
 
+class CalibrationEntryMissingException(CalibrationException):
+    """Raised when filter_by_parameter cannot locate calibration data."""
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+
+class CalibrationParametersMissingException(CalibrationException):
+    """Raised when a dictionary does not contain all calibration parameters as keys."""
+
+    def __init__(self, provided_dict: dict, required_keys: list):
+        msg = (
+            "Missing required parameters to lookup calibration data.\n"
+            + f"Required parameters are {required_keys}\n"
+            + f"Provided parameters are {list(provided_dict.keys())}"
+        )
+        super().__init__(msg)
+
+
 def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> dict:
     """
     Select a certain element by the value of a top-level key in a dictionary.
@@ -42,16 +61,16 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
             raise KeyError
         else:
             return filtered_data
-    except AttributeError as e:
+    except AttributeError:
         # calibrations does not have ".get()"
         # Generally means that calibrations is None or not a dict
         msg = f"Provided calibration data is not a dict: {calibrations}"
         raise CalibrationException(msg)
-    except KeyError as e:
+    except KeyError:
         msg = (
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
             + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )
-        raise CalibrationException(msg)
+        raise CalibrationEntryMissingException(msg)
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 6143b0a5..94aca3c3 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -2,7 +2,7 @@
 import hashlib
 import json
 import logging
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from its_preselector.preselector import Preselector
 from its_preselector.web_relay import WebRelay
@@ -10,10 +10,7 @@
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.gps_iface import GPSInterface
-from scos_actions.hardware.sigan_iface import (
-    SIGAN_SETTINGS_KEYS,
-    SignalAnalyzerInterface,
-)
+from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import convert_string_to_millisecond_iso_format
 
 logger = logging.getLogger(__name__)
@@ -143,63 +140,37 @@ def last_calibration_time(self) -> str:
     @property
     def sensor_calibration_data(self) -> Dict[str, Any]:
         """Sensor calibration data for the current sensor settings."""
-        self._recompute_sensor_calibration_data()
         return self._sensor_calibration_data
 
     @property
     def differential_calibration_data(self) -> Dict[str, float]:
         """Differential calibration data for the current sensor settings."""
-        self._recompute_differential_calibration_data()
         return self._differential_calibration_data
 
-    def _get_calibration_args_from_sigan(self, calibration: Calibration) -> list:
-        """Get current values of signal analyzer settings which are calibration parameters."""
-        try:
-            # Get calibration parameters which are valid settings for signal analyzers
-            cal_params = [
-                p
-                for p in calibration.calibration_parameters
-                if p in SIGAN_SETTINGS_KEYS
-            ]
-            if len(cal_params) == 0:
-                err_text = "any"  # Formats error message below
-                raise ValueError
-            else:
-                err_text = "this"  # Formats error message below
-            cal_args = [vars(self.signal_analyzer)[f"_{p}"] for p in cal_params]
-        except Exception as e:
-            msg = (
-                f"One or more calibration parameters is not a valid setting for {err_text} "
-                + f"signal analyzer.\nRequired parameters: {calibration.calibration_parameters}"
-            )
-            logger.exception(msg)
-            raise e
-        logger.debug(f"Matched calibration params: {cal_args}")
-        return cal_args  # Order matches calibration.calibration_parameters
-
-    def _recompute_differential_calibration_data(self) -> None:
-        """Set the differential calibration data based on the current tuning."""
-        self._differential_calibration_data = {}
+    def recompute_calibration_data(self, params: dict) -> None:
+        """
+        Set the differential_calibration_data and sensor_calibration_data
+        based on the specified ``params``.
+        """
+        recomputed = False
         if self.differential_calibration is not None:
-            cal_args = self._get_calibration_args_from_sigan(
-                self.differential_calibration
-            )
             self._differential_calibration_data.update(
-                self.differential_calibration.get_calibration_dict(cal_args)
+                self.differential_calibration.get_calibration_dict(params)
             )
+            recomputed = True
         else:
-            logger.warning("Differential calibration does not exist.")
+            logger.debug("No differential calibration available to recompute")
 
-    def _recompute_sensor_calibration_data(self) -> None:
-        """Set the sensor calibration data based on the current tuning."""
-        self._sensor_calibration_data = {}
         if self.sensor_calibration is not None:
-            cal_args = self._get_calibration_args_from_sigan(self.sensor_calibration)
             self._sensor_calibration_data.update(
-                self.sensor_calibration.get_calibration_dict(cal_args)
+                self.sensor_calibration.get_calibration_dict(params)
             )
+            recomputed = True
         else:
-            logger.warning("Sensor calibration does not exist.")
+            logger.debug("No sensor calibration available to recompute")
+
+        if not recomputed:
+            logger.warning("Failed to recompute calibration data")
 
     def acquire_time_domain_samples(
         self,
@@ -207,6 +178,7 @@ def acquire_time_domain_samples(
         num_samples_skip: int = 0,
         retries: int = 5,
         cal_adjust: bool = True,
+        cal_params: Optional[dict] = None,
     ) -> dict:
         """
         Acquire time-domain IQ samples from the signal analyzer.
@@ -224,6 +196,9 @@ def acquire_time_domain_samples(
         :param num_samples_skip: Number of samples to skip
         :param retries: Maximum number of retries on failure
         :param cal_adjust: If True, use available calibration data to scale the samples.
+        :param cal_params: A dictionary with keys for all of the calibration parameters.
+            May contain additional keys. Example: ``{"sample_rate": 14000000.0, "gain": 10.0}``
+            Must be specified if ``cal_adjust`` is ``True``. Otherwise, ignored.
         :return: dictionary containing data, sample_rate, frequency, capture_time, etc
         :raises Exception: If the sample acquisition fails, or the sensor has
             no signal analyzer.
@@ -231,7 +206,6 @@ def acquire_time_domain_samples(
         logger.debug("Sensor.acquire_time_domain_samples starting")
         logger.debug(f"Number of retries = {retries}")
         max_retries = retries
-        # TODO: Include RF path as a sensor cal argument?
         # Acquire samples from signal analyzer
         if self.signal_analyzer is not None:
             while True:
@@ -258,15 +232,18 @@ def acquire_time_domain_samples(
 
         # Apply gain adjustment based on calibration
         if cal_adjust:
+            if cal_params is None:
+                raise ValueError(
+                    "Data scaling cannot occur without specified calibration parameters."
+                )
             if self.sensor_calibration is not None:
                 logger.debug("Scaling samples using calibration data")
+                self.recompute_calibration_data(cal_params)
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
-                    # TODO recompute functions match to current signal analyzer
-                    # settings. should they use the measurement_result instead?
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss
@@ -297,6 +274,6 @@ def acquire_time_domain_samples(
         else:
             # Set the data reference in the measurement_result
             measurement_result["reference"] = "signal analyzer input"
-            measurement_result["calibration"] = None
+            measurement_result["applied_calibration"] = None
 
         return measurement_result

From 087a3c1690841fb3587caab882ddaf406cde4952 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 18:48:23 -0500
Subject: [PATCH 27/62] update DifferentialCalibration docstring

---
 .../calibration/differential_calibration.py   | 25 ++++++++-----------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/scos_actions/calibration/differential_calibration.py b/scos_actions/calibration/differential_calibration.py
index e25ad71d..ac518737 100644
--- a/scos_actions/calibration/differential_calibration.py
+++ b/scos_actions/calibration/differential_calibration.py
@@ -2,19 +2,16 @@
 Dataclass implementation for "differential calibration" handling.
 
 A differential calibration provides loss values which represent excess loss
-between the `SensorCalibration` reference point and another reference point.
-A typical usage would be for calibrating out measured cable losses which exist
-between the antenna and the Y-factor calibration terminal. At present, this is
-measured manually using a calibration probe consisting of a calibrated noise
-source and a programmable attenuator.
-
-The ``reference_point`` top-level key defines the point to which measurements
-are referenced after using the correction factors included in the file.
-
-The ``calibration_data`` entries are expected to include these correction factors,
-with the key name ``"loss"`` and values in decibels (dB). A positive value of
-``"loss"`` indicates a LOSS going FROM ``reference_point`` TO the calibration
-terminal used by the ``SensorCalibration``.
+between the ``SensorCalibration.calibration_reference`` reference point and
+another reference point. A typical usage would be for calibrating out measured
+cable losses which exist between the antenna and the Y-factor calibration terminal.
+At present, this is measured manually using a calibration probe consisting of a
+calibrated noise source and a programmable attenuator.
+
+The ``DifferentialCalibration.calibration_data`` entries should be dictionaries
+containing the key ``"loss"`` and a corresponding value in decibels (dB). A positive
+value of ``"loss"`` indicates a LOSS going FROM ``DifferentialCalibration.calibration_reference``
+TO ``SensorCalibration.calibration_reference``.
 """
 
 from dataclasses import dataclass
@@ -24,8 +21,6 @@
 
 @dataclass
 class DifferentialCalibration(Calibration):
-    reference_point: str
-
     def update(self):
         """
         SCOS Sensor should not update differential calibration files.

From 5c99a97aa3426a7f50b977848bfb809c8caa0a12 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Thu, 25 Jan 2024 18:57:38 -0500
Subject: [PATCH 28/62] make calibration_reference required for all
 calibrations

---
 scos_actions/calibration/interfaces/calibration.py |  4 ++++
 scos_actions/calibration/sensor_calibration.py     |  3 +++
 scos_actions/calibration/tests/test_calibration.py | 14 ++++++++++----
 .../tests/test_differential_calibration.py         |  4 ++--
 .../calibration/tests/test_sensor_calibration.py   |  4 ++++
 scos_actions/hardware/sensor.py                    |  4 ++--
 6 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 42277253..6f1354ef 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -1,3 +1,6 @@
+"""
+TODO
+"""
 import dataclasses
 import json
 import logging
@@ -17,6 +20,7 @@
 class Calibration:
     calibration_parameters: List[str]
     calibration_data: dict
+    calibration_reference: str
     is_default: bool
     file_path: Path
 
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index de547fc9..a4f23a54 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -1,3 +1,6 @@
+"""
+TODO
+"""
 import logging
 from dataclasses import dataclass
 from typing import Dict, List, Union
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 9fb181e4..1e19b127 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -34,6 +34,7 @@ def setup_calibration_file(self, tmp_path: Path):
         self.sample_cal = Calibration(
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=self.dummy_file_path,
         )
@@ -41,6 +42,7 @@ def setup_calibration_file(self, tmp_path: Path):
         self.sample_default_cal = Calibration(
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
+            calibration_reference="testing",
             is_default=True,
             file_path=self.dummy_default_file_path,
         )
@@ -56,6 +58,7 @@ def test_calibration_dataclass_fields(self):
         # Note: does not check field order
         assert fields == {
             "calibration_parameters": List[str],
+            "calibration_reference": str,
             "is_default": bool,
             "calibration_data": dict,
             "file_path": Path,
@@ -64,13 +67,15 @@ def test_calibration_dataclass_fields(self):
     def test_field_validator(self):
         """Check that the input field type validator works."""
         with pytest.raises(TypeError):
-            _ = Calibration([], {}, False, False)
+            _ = Calibration([], {}, "", False, False)
         with pytest.raises(TypeError):
-            _ = Calibration([], {}, 100, Path(""))
+            _ = Calibration([], {}, "", 100, Path(""))
         with pytest.raises(TypeError):
-            _ = Calibration([], [10, 20], False, Path(""))
+            _ = Calibration([], {}, 5, False, Path(""))
         with pytest.raises(TypeError):
-            _ = Calibration({"test": 1}, {}, False, Path(""))
+            _ = Calibration([], [10, 20], "", False, Path(""))
+        with pytest.raises(TypeError):
+            _ = Calibration({"test": 1}, {}, "", False, Path(""))
 
     def test_get_calibration_dict(self):
         """Check the get_calibration_dict method with all dummy data."""
@@ -105,6 +110,7 @@ def test_to_and_from_json(self, tmp_path: Path):
         sensor_cal = SensorCalibration(
             self.sample_cal.calibration_parameters,
             self.sample_cal.calibration_data,
+            "testing",
             False,
             tmp_path / "testing.json",
             "dt_str",
diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
index 8f1b5594..fe53dea9 100644
--- a/scos_actions/calibration/tests/test_differential_calibration.py
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -11,7 +11,7 @@ class TestDifferentialCalibration:
     def setup_differential_calibration_file(self, tmp_path: Path):
         dict_to_json = {
             "calibration_parameters": ["frequency"],
-            "reference_point": "antenna input",
+            "calibration_reference": "antenna input",
             "calibration_data": {3555e6: 11.5},
         }
         self.valid_file_path = tmp_path / "sample_diff_cal.json"
@@ -24,7 +24,7 @@ def setup_differential_calibration_file(self, tmp_path: Path):
         with open(self.valid_file_path, "w") as f:
             f.write(json.dumps(dict_to_json))
 
-        dict_to_json.pop("reference_point", None)
+        dict_to_json.pop("calibration_reference", None)
 
         with open(self.invalid_file_path, "w") as f:
             f.write(json.dumps(dict_to_json))
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index e9176718..f4433321 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -138,6 +138,7 @@ def setup_calibration_file(self, tmp_path: Path):
         # Add the simple stuff to new cal format
         cal_data["last_calibration_datetime"] = get_datetime_str_now()
         cal_data["sensor_uid"] = "SAMPLE_CALIBRATION"
+        cal_data["calibration_reference"] = "TESTING"
 
         # Add SR/CF lookup table
         cal_data["clock_rate_lookup_by_sample_rate"] = []
@@ -245,6 +246,7 @@ def test_get_calibration_dict_exact_match_lookup(self):
         cal = SensorCalibration(
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=Path(""),
             last_calibration_datetime=calibration_datetime,
@@ -264,6 +266,7 @@ def test_get_calibration_dict_within_range(self):
         cal = SensorCalibration(
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=Path("test_calibration.json"),
             last_calibration_datetime=calibration_datetime,
@@ -306,6 +309,7 @@ def test_update(self):
         cal = SensorCalibration(
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
+            calibration_reference="testing",
             is_default=False,
             file_path=test_cal_path,
             last_calibration_datetime=calibration_datetime,
diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 94aca3c3..b95adfcf 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -242,6 +242,7 @@ def acquire_time_domain_samples(
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
+                measurement_result = self.sensor_calibration.calibration_reference
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     differential_loss = self.differential_calibration_data["loss"]
@@ -250,11 +251,10 @@ def acquire_time_domain_samples(
                     calibrated_nf__db += differential_loss
                     measurement_result[
                         "reference"
-                    ] = self.differential_calibration.reference_point
+                    ] = self.differential_calibration.calibration_reference
                 else:
                     # No differential calibration exists
                     logger.debug("No differential calibration was applied")
-                    measurement_result["reference"] = "calibration terminal"
 
                 linear_gain = 10.0 ** (calibrated_gain__db / 20.0)
                 logger.debug(f"Applying total gain of {calibrated_gain__db}")

From 4990d2eeefd078417d5064f6dc84bc9dd28aac3a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 7 Feb 2024 16:30:45 -0500
Subject: [PATCH 29/62] fail early if sensor has no calibration object

---
 scos_actions/actions/calibrate_y_factor.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index bf1099e7..b777e7ac 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -75,6 +75,8 @@
 import numpy as np
 from scipy.constants import Boltzmann
 from scipy.signal import sosfilt
+
+from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS
@@ -92,8 +94,6 @@
 from scos_actions.signals import trigger_api_restart
 from scos_actions.utils import ParameterException, get_parameter
 
-from scos_actions import utils
-
 logger = logging.getLogger(__name__)
 
 # Define parameter keys
@@ -196,6 +196,8 @@ def __init__(self, parameters: dict):
     def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         """This is the entrypoint function called by the scheduler."""
         self.sensor = sensor
+        if self.sensor.sensor_calibration is None:
+            raise Exception("Sensor object must have a SensorCalibration object")
         self.test_required_components()
         detail = ""
 

From 17595faddd19d229da21cae686ce7cb56f47035a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 7 Feb 2024 17:02:15 -0500
Subject: [PATCH 30/62] fix missing indexing key

---
 scos_actions/hardware/sensor.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 62884d1a..b5178bf7 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -316,7 +316,9 @@ def acquire_time_domain_samples(
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
-                measurement_result = self.sensor_calibration.calibration_reference
+                measurement_result[
+                    "reference"
+                ] = self.sensor_calibration.calibration_reference
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     differential_loss = self.differential_calibration_data["loss"]

From 2ddaecc07c368f1ee70cbdaeb3c389ddea5f1d97 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 26 Feb 2024 16:21:24 -0500
Subject: [PATCH 31/62] Update .pre-commit-config.yaml

---
 .pre-commit-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 9618961d..12ef9bfc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.15.1
     hooks:
       - id: pyupgrade
         args: ["--py38-plus"]
@@ -30,12 +30,12 @@ repos:
         types: [file, python]
         args: ["--profile", "black", "--filter-files", "--gitignore"]
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.2.0
     hooks:
       - id: black
         types: [file, python]
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.38.0
+    rev: v0.39.0
     hooks:
       - id: markdownlint
         types: [file, markdown]

From 88a6b32c2388398726340d3dbf2fa90c45df1477 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 26 Feb 2024 16:23:14 -0500
Subject: [PATCH 32/62] fix log message when checking integer keys

---
 scos_actions/calibration/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index 6ddba00b..9370c068 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -70,7 +70,7 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
         msg = (
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f'and {float(value)}' if isinstance(value, int) else ''}"
+            + f"{f'and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )
         raise CalibrationEntryMissingException(msg)

From 382e9075563c999a3d27fcacac1f5387ade574ce Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Mon, 26 Feb 2024 16:31:33 -0500
Subject: [PATCH 33/62] add debug mesasges for testing

---
 scos_actions/hardware/sensor.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index b5178bf7..3e7552c0 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -277,8 +277,15 @@ def acquire_time_domain_samples(
         :raises Exception: If the sample acquisition fails, or the sensor has
             no signal analyzer.
         """
+        logger.debug("***********************************\n")
         logger.debug("Sensor.acquire_time_domain_samples starting")
         logger.debug(f"Number of retries = {retries}")
+        logger.debug(
+            f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
+        )
+        logger.debug(f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}")
+        logger.debug("*************************************\n")
+
         max_retries = retries
         # Acquire samples from signal analyzer
         if self.signal_analyzer is not None:
@@ -316,18 +323,18 @@ def acquire_time_domain_samples(
                 calibrated_gain__db = self.sensor_calibration_data["gain"]
                 calibrated_nf__db = self.sensor_calibration_data["noise_figure"]
                 logger.debug(f"Using sensor gain: {calibrated_gain__db} dB")
-                measurement_result[
-                    "reference"
-                ] = self.sensor_calibration.calibration_reference
+                measurement_result["reference"] = (
+                    self.sensor_calibration.calibration_reference
+                )
                 if self.differential_calibration is not None:
                     # Also apply differential calibration correction
                     differential_loss = self.differential_calibration_data["loss"]
                     logger.debug(f"Using differential loss: {differential_loss} dB")
                     calibrated_gain__db -= differential_loss
                     calibrated_nf__db += differential_loss
-                    measurement_result[
-                        "reference"
-                    ] = self.differential_calibration.calibration_reference
+                    measurement_result["reference"] = (
+                        self.differential_calibration.calibration_reference
+                    )
                 else:
                     # No differential calibration exists
                     logger.debug("No differential calibration was applied")

From 89313d84092a97bbbea4634546ea048672fa05b0 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 12:58:08 -0500
Subject: [PATCH 34/62] remove is_default calibration parameter

---
 .../calibration/interfaces/calibration.py         | 15 ++++++---------
 .../calibration/tests/test_calibration.py         | 14 ++------------
 .../tests/test_differential_calibration.py        |  4 +++-
 .../calibration/tests/test_sensor_calibration.py  |  5 ++---
 4 files changed, 13 insertions(+), 25 deletions(-)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index 6f1354ef..b243bea4 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -1,6 +1,7 @@
 """
 TODO
 """
+
 import dataclasses
 import json
 import logging
@@ -21,7 +22,6 @@ class Calibration:
     calibration_parameters: List[str]
     calibration_data: dict
     calibration_reference: str
-    is_default: bool
     file_path: Path
 
     def __post_init__(self):
@@ -75,7 +75,7 @@ def update(self):
         raise NotImplementedError
 
     @classmethod
-    def from_json(cls, fname: Path, is_default: bool):
+    def from_json(cls, fname: Path):
         """
         Load a calibration from a JSON file.
 
@@ -84,8 +84,6 @@ def from_json(cls, fname: Path, is_default: bool):
         the class being constructed.
 
         :param fname: The ``Path`` to the JSON calibration file.
-        :param is_default: If True, the loaded calibration file
-            is treated as the default calibration file.
         :raises Exception: If the provided file does not include
             the required keys.
         :return: The ``Calibration`` object generated from the file.
@@ -96,7 +94,7 @@ def from_json(cls, fname: Path, is_default: bool):
 
         # Check that only the required fields are in the dict
         required_keys = {f.name for f in dataclasses.fields(cls)}
-        required_keys -= {"is_default", "file_path"}  # are not required in JSON
+        required_keys -= {"file_path"}  # not required in JSON
         if cal_file_keys == required_keys:
             pass
         elif cal_file_keys >= required_keys:
@@ -115,7 +113,7 @@ def from_json(cls, fname: Path, is_default: bool):
             )
 
         # Create and return the Calibration object
-        return cls(is_default=is_default, file_path=fname, **calibration)
+        return cls(file_path=fname, **calibration)
 
     def to_json(self) -> None:
         """
@@ -123,12 +121,11 @@ def to_json(self) -> None:
 
         The JSON file will be located at ``self.file_path`` and will
         contain a copy of ``self.__dict__``, except for the ``file_path``
-        and ``is_default`` key/value pairs. This includes all dataclass
-        fields, with their parameter names as JSON key names.
+        key/value pair. This includes all dataclass fields, with their
+        parameter names as JSON key names.
         """
         dict_to_json = self.__dict__.copy()
         # Remove keys which should not save to JSON
         dict_to_json.pop("file_path", None)
-        dict_to_json.pop("is_default", None)
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(dict_to_json))
diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 1e19b127..297808ef 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -1,10 +1,12 @@
 """Test the Calibration base dataclass."""
+
 import dataclasses
 import json
 from pathlib import Path
 from typing import List
 
 import pytest
+
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.calibration.tests.utils import recursive_check_keys
@@ -35,7 +37,6 @@ def setup_calibration_file(self, tmp_path: Path):
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=self.dummy_file_path,
         )
 
@@ -43,7 +44,6 @@ def setup_calibration_file(self, tmp_path: Path):
             calibration_parameters=self.cal_params,
             calibration_data=self.cal_data,
             calibration_reference="testing",
-            is_default=True,
             file_path=self.dummy_default_file_path,
         )
 
@@ -59,7 +59,6 @@ def test_calibration_dataclass_fields(self):
         assert fields == {
             "calibration_parameters": List[str],
             "calibration_reference": str,
-            "is_default": bool,
             "calibration_data": dict,
             "file_path": Path,
         }, "Calibration class fields have changed"
@@ -96,15 +95,6 @@ def test_to_and_from_json(self, tmp_path: Path):
             self.dummy_default_file_path, True
         )
 
-        # These should fail: the is_default parameter is different
-        # even though the other contents are identical.
-        with pytest.raises(AssertionError):
-            assert self.sample_cal == Calibration.from_json(self.dummy_file_path, True)
-        with pytest.raises(AssertionError):
-            assert self.sample_default_cal == Calibration.from_json(
-                self.dummy_default_file_path, False
-            )
-
         # from_json should ignore extra keys in the loaded file, but not fail
         # Test this by trying to load a SensorCalibration as a Calibration
         sensor_cal = SensorCalibration(
diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
index fe53dea9..88025a82 100644
--- a/scos_actions/calibration/tests/test_differential_calibration.py
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -1,8 +1,10 @@
 """Test the DifferentialCalibration dataclass."""
+
 import json
 from pathlib import Path
 
 import pytest
+
 from scos_actions.calibration.differential_calibration import DifferentialCalibration
 
 
@@ -18,7 +20,7 @@ def setup_differential_calibration_file(self, tmp_path: Path):
         self.invalid_file_path = tmp_path / "sample_diff_cal_invalid.json"
 
         self.sample_diff_cal = DifferentialCalibration(
-            is_default=False, file_path=self.valid_file_path, **dict_to_json
+            file_path=self.valid_file_path, **dict_to_json
         )
 
         with open(self.valid_file_path, "w") as f:
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index f4433321..86c0fdd7 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -1,4 +1,5 @@
 """Test the SensorCalibration dataclass."""
+
 import dataclasses
 import datetime
 import json
@@ -9,6 +10,7 @@
 from typing import Dict, List
 
 import pytest
+
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.calibration.tests.utils import recursive_check_keys
@@ -247,7 +249,6 @@ def test_get_calibration_dict_exact_match_lookup(self):
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=Path(""),
             last_calibration_datetime=calibration_datetime,
             clock_rate_lookup_by_sample_rate=[],
@@ -267,7 +268,6 @@ def test_get_calibration_dict_within_range(self):
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=Path("test_calibration.json"),
             last_calibration_datetime=calibration_datetime,
             clock_rate_lookup_by_sample_rate=[],
@@ -310,7 +310,6 @@ def test_update(self):
             calibration_parameters=calibration_params,
             calibration_data=calibration_data,
             calibration_reference="testing",
-            is_default=False,
             file_path=test_cal_path,
             last_calibration_datetime=calibration_datetime,
             clock_rate_lookup_by_sample_rate=[],

From 785149388c07c75aaabc594026ff49d54a0d1960 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:10:10 -0500
Subject: [PATCH 35/62] Do not overwrite sensor calibration file

---
 scos_actions/actions/calibrate_y_factor.py | 64 +++++++++++++++++++---
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index b777e7ac..f3a75f4d 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -17,7 +17,13 @@
 # - SCOS Markdown Editor: https://ntia.github.io/scos-md-editor/
 #
 r"""Perform a Y-Factor Calibration.
-Supports calibration of gain and noise figure for one or more channels.
+Supports calculation of gain and noise figure for one or more channels using the
+Y-Factor method. Results are written to the file specified by the environment
+variable ``ONBOARD_CALIBRATION_FILE``. If the sensor already has a sensor calibration
+object, it is used as the starting point, and copied to a new onboard calibration object
+which is updated by this action. The sensor object's sensor calibration will be set to
+the updated onboard calibration object after this action is run.
+
 For each center frequency, sets the preselector to the noise diode path, turns
 noise diode on, performs a mean power measurement, turns the noise diode off and
 performs another mean power measurement. The mean power on and mean power off
@@ -73,11 +79,13 @@
 import time
 
 import numpy as np
+from environs import Env
 from scipy.constants import Boltzmann
 from scipy.signal import sosfilt
 
 from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
+from scos_actions.calibration.sensor_calibration import SensorCalibration
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS
 from scos_actions.signal_processing.calibration import (
@@ -92,9 +100,10 @@
 from scos_actions.signal_processing.power_analysis import calculate_power_watts
 from scos_actions.signal_processing.unit_conversion import convert_watts_to_dBm
 from scos_actions.signals import trigger_api_restart
-from scos_actions.utils import ParameterException, get_parameter
+from scos_actions.utils import ParameterException, get_datetime_str_now, get_parameter
 
 logger = logging.getLogger(__name__)
+env = Env()
 
 # Define parameter keys
 RF_PATH = Action.PRESELECTOR_PATH_KEY
@@ -112,6 +121,7 @@
 IIR_RESP_FREQS = "iir_num_response_frequencies"
 CAL_SOURCE_IDX = "cal_source_idx"
 TEMP_SENSOR_IDX = "temp_sensor_idx"
+REFERENCE_POINT = "reference_point"
 
 
 class YFactorCalibration(Action):
@@ -196,8 +206,44 @@ def __init__(self, parameters: dict):
     def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         """This is the entrypoint function called by the scheduler."""
         self.sensor = sensor
+
+        # Prepare the sensor calibration object.
+        assert all(
+            self.iteration_params[0][REFERENCE_POINT] == p[REFERENCE_POINT]
+            for p in self.iteration_params
+        ), f"All iterations must use the same '{REFERENCE_POINT}' setting"
+        onboard_cal_reference = self.iteration_params[0][REFERENCE_POINT]
+
         if self.sensor.sensor_calibration is None:
-            raise Exception("Sensor object must have a SensorCalibration object")
+            # Create a new sensor calibration object and attach it to the sensor.
+            # The calibration parameters will be set to the sigan parameters used
+            # in the action YAML parameters.
+            logger.debug(f"Creating a new onboard cal object for the sensor.")
+            cal_params = [k for k in self.iteration_params if k in SIGAN_SETTINGS_KEYS]
+            cal_data = dict()
+            last_cal_datetime = get_datetime_str_now()
+            clock_rate_lookup_by_sample_rate = []
+            sensor_uid = "Sensor calibration file not provided"
+            self.sensor.sensor_calibration = SensorCalibration(
+                calibration_parameters=cal_params,
+                calibration_data=cal_data,
+                calibration_reference=onboard_cal_reference,
+                file_path=env("ONBOARD_CALIBRATION_FILE"),
+                last_calibration_datetime=last_cal_datetime,
+                clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
+                sensor_uid=sensor_uid,
+            )
+        elif self.sensor.sensor_calibration.file_path == env(
+            "ONBOARD_CALIBRATION_FILE"
+        ):
+            # Already using an onboard cal file.
+            logger.debug("Onboard calibration file already in use. Continuing.")
+        else:
+            # Sensor calibration file exists. Change it to an onboard cal file
+            logger.debug("Making new onboard cal file from existing sensor cal")
+            self.sensor.sensor_calibration.calibration_reference = onboard_cal_reference
+            self.sensor.sensor_calibration.file_path = env("ONBOARD_CALIBRATION_FILE")
+
         self.test_required_components()
         detail = ""
 
@@ -207,6 +253,8 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 detail += self.calibrate(p)
             else:
                 detail += os.linesep + self.calibrate(p)
+        # Save results to onboard calibration file
+        self.sensor.sensor_calibration.to_json()
         return detail
 
     def calibrate(self, params: dict):
@@ -257,11 +305,6 @@ def calibrate(self, params: dict):
             noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"])
             noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"])
         else:
-            if self.sensor.sensor_calibration.is_default:
-                raise Exception(
-                    "Calibrations without IIR filter cannot be performed with default calibration."
-                )
-
             logger.debug("Skipping IIR filtering")
             # Get ENBW from sensor calibration
             assert set(self.sensor.sensor_calibration.calibration_parameters) <= set(
@@ -272,6 +315,11 @@ def calibrate(self, params: dict):
                 for k in self.sensor.sensor_calibration.calibration_parameters
             ]
             self.sensor.recompute_sensor_calibration_data(cal_args)
+            if "enbw" not in self.sensor.sensor_calibration_data:
+                raise Exception(
+                    "Unable to perform Y-Factor calibration without IIR filtering when no"
+                    " ENBW is provided in the sensor calibration file."
+                )
             enbw_hz = self.sensor.sensor_calibration_data["enbw"]
             noise_on_data = noise_on_measurement_result["data"]
             noise_off_data = noise_off_measurement_result["data"]

From 51820e8508795fcf89752b179b7479c6ba25e259 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:25:07 -0500
Subject: [PATCH 36/62] fix environment variable reference

---
 scos_actions/actions/calibrate_y_factor.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index f3a75f4d..a1593b12 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -19,7 +19,7 @@
 r"""Perform a Y-Factor Calibration.
 Supports calculation of gain and noise figure for one or more channels using the
 Y-Factor method. Results are written to the file specified by the environment
-variable ``ONBOARD_CALIBRATION_FILE``. If the sensor already has a sensor calibration
+variable ``ONBOARD_SENSOR_CALIBRATION_FILE``. If the sensor already has a sensor calibration
 object, it is used as the starting point, and copied to a new onboard calibration object
 which is updated by this action. The sensor object's sensor calibration will be set to
 the updated onboard calibration object after this action is run.
@@ -228,13 +228,13 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 calibration_parameters=cal_params,
                 calibration_data=cal_data,
                 calibration_reference=onboard_cal_reference,
-                file_path=env("ONBOARD_CALIBRATION_FILE"),
+                file_path=env("ONBOARD_SENSOR_CALIBRATION_FILE"),
                 last_calibration_datetime=last_cal_datetime,
                 clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
                 sensor_uid=sensor_uid,
             )
         elif self.sensor.sensor_calibration.file_path == env(
-            "ONBOARD_CALIBRATION_FILE"
+            "ONBOARD_SENSOR_CALIBRATION_FILE"
         ):
             # Already using an onboard cal file.
             logger.debug("Onboard calibration file already in use. Continuing.")

From afbf2bf4837b8e25e1da2ec84c8f157829480758 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:31:07 -0500
Subject: [PATCH 37/62] add missing import

---
 scos_actions/hardware/mocks/mock_sigan.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py
index ae675556..b4394e06 100644
--- a/scos_actions/hardware/mocks/mock_sigan.py
+++ b/scos_actions/hardware/mocks/mock_sigan.py
@@ -1,10 +1,12 @@
 """Mock a signal analyzer for testing."""
+
 import logging
 from collections import namedtuple
 from typing import Optional
 
 import numpy as np
 
+from scos_actions import __version__ as SCOS_ACTIONS_VERSION
 from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface
 from scos_actions.utils import get_datetime_str_now
 

From 8bddcf2b9d0465d7c502b3650e091dc3487d29a3 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:35:53 -0500
Subject: [PATCH 38/62] fix from_json calibration file tests

---
 .../calibration/tests/test_differential_calibration.py        | 4 ++--
 scos_actions/calibration/tests/test_sensor_calibration.py     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scos_actions/calibration/tests/test_differential_calibration.py b/scos_actions/calibration/tests/test_differential_calibration.py
index 88025a82..5c9c80ca 100644
--- a/scos_actions/calibration/tests/test_differential_calibration.py
+++ b/scos_actions/calibration/tests/test_differential_calibration.py
@@ -33,10 +33,10 @@ def setup_differential_calibration_file(self, tmp_path: Path):
 
     def test_from_json(self):
         """Check from_json functionality with valid and invalid dummy data."""
-        diff_cal = DifferentialCalibration.from_json(self.valid_file_path, False)
+        diff_cal = DifferentialCalibration.from_json(self.valid_file_path)
         assert diff_cal == self.sample_diff_cal
         with pytest.raises(Exception):
-            _ = DifferentialCalibration.from_json(self.invalid_file_path, False)
+            _ = DifferentialCalibration.from_json(self.invalid_file_path)
 
     def test_update_not_implemented(self):
         """Check that the update method is not implemented."""
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 86c0fdd7..d8914a68 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -184,7 +184,7 @@ def setup_calibration_file(self, tmp_path: Path):
             json.dump(cal_data, file, indent=4)
 
         # Load the data back in
-        self.sample_cal = SensorCalibration.from_json(self.calibration_file, False)
+        self.sample_cal = SensorCalibration.from_json(self.calibration_file)
 
         # Create a list of previous points to ensure that we don't repeat
         self.pytest_points = []
@@ -318,7 +318,7 @@ def test_update(self):
         action_params = {"sample_rate": 100.0, "frequency": 200.0}
         update_time = get_datetime_str_now()
         cal.update(action_params, update_time, 30.0, 5.0, 21)
-        cal_from_file = SensorCalibration.from_json(test_cal_path, False)
+        cal_from_file = SensorCalibration.from_json(test_cal_path)
         test_cal_path.unlink()
         file_utc_time = parse_datetime_iso_format_str(cal.last_calibration_datetime)
         cal_time_utc = parse_datetime_iso_format_str(update_time)

From c36efd84e86bb8850af5b051da401bc0f610bc19 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:38:59 -0500
Subject: [PATCH 39/62] remove unused imports

---
 scos_actions/actions/acquire_single_freq_fft.py       |  8 ++++----
 .../actions/acquire_single_freq_tdomain_iq.py         | 11 +++++------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index 21bc8f5e..3ae3fe7a 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -89,8 +89,8 @@
 import logging
 
 from numpy import float32, ndarray
+
 from scos_actions.actions.interfaces.measurement_action import MeasurementAction
-from scos_actions.hardware.mocks.mock_gps import MockGPS
 from scos_actions.metadata.structs import ntia_algorithm
 from scos_actions.signal_processing.fft import (
     get_fft,
@@ -173,9 +173,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Save measurement results
         measurement_result["data"] = m4s_result
         measurement_result.update(self.parameters)
-        measurement_result[
-            "calibration_datetime"
-        ] = self.sensor.sensor_calibration_data["datetime"]
+        measurement_result["calibration_datetime"] = (
+            self.sensor.sensor_calibration_data["datetime"]
+        )
         measurement_result["task_id"] = task_id
         measurement_result["classification"] = self.classification
 
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 6ff6b537..14796a05 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -34,11 +34,10 @@
 import logging
 
 from numpy import complex64
-from scos_actions.actions.interfaces.measurement_action import MeasurementAction
-from scos_actions.hardware.mocks.mock_gps import MockGPS
-from scos_actions.utils import get_parameter
 
 from scos_actions import utils
+from scos_actions.actions.interfaces.measurement_action import MeasurementAction
+from scos_actions.utils import get_parameter
 
 logger = logging.getLogger(__name__)
 
@@ -91,9 +90,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result.update(self.parameters)
         measurement_result["end_time"] = end_time
         measurement_result["task_id"] = task_id
-        measurement_result[
-            "calibration_datetime"
-        ] = self.sensor.sensor_calibration_data["datetime"]
+        measurement_result["calibration_datetime"] = (
+            self.sensor.sensor_calibration_data["datetime"]
+        )
         measurement_result["classification"] = self.classification
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")

From 57a3dca3cc03a3c20f12218cc3806a6396a41ea5 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:43:37 -0500
Subject: [PATCH 40/62] fix cal data lookup unit tests

---
 scos_actions/calibration/tests/test_sensor_calibration.py | 2 +-
 scos_actions/calibration/tests/test_utils.py              | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index d8914a68..743faf1b 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -277,7 +277,7 @@ def test_get_calibration_dict_within_range(self):
             _ = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 250.0})
         assert e_info.value.args[0] == (
             f"Could not locate calibration data at 250.0"
-            + f"\nAttempted lookup using key '250.0'"
+            + f"\nAttempted lookup using key '250.0' and 250.0"
             + f"\nUsing calibration data: {cal.calibration_data['100.0']}"
         )
 
diff --git a/scos_actions/calibration/tests/test_utils.py b/scos_actions/calibration/tests/test_utils.py
index 2a751831..15ab29cf 100644
--- a/scos_actions/calibration/tests/test_utils.py
+++ b/scos_actions/calibration/tests/test_utils.py
@@ -1,4 +1,5 @@
 import pytest
+
 from scos_actions.calibration.utils import CalibrationException, filter_by_parameter
 
 
@@ -10,7 +11,7 @@ def test_filter_by_parameter_out_of_range(self):
         assert (
             e_info.value.args[0]
             == f"Could not locate calibration data at 400.0"
-            + f"\nAttempted lookup using key '400.0'"
+            + f"\nAttempted lookup using key '400.0' and 400.0"
             + f"\nUsing calibration data: {calibrations}"
         )
 
@@ -23,7 +24,7 @@ def test_filter_by_parameter_in_range_requires_match(self):
             _ = filter_by_parameter(calibrations, 150.0)
         assert e_info.value.args[0] == (
             f"Could not locate calibration data at 150.0"
-            + f"\nAttempted lookup using key '150.0'"
+            + f"\nAttempted lookup using key '150.0' and 150.0"
             + f"\nUsing calibration data: {calibrations}"
         )
 

From 3d52f8e41317048bcc5011bb9d9c0867f9fd1fd4 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 5 Mar 2024 17:44:37 -0500
Subject: [PATCH 41/62] fix formatting in cal lookup error

---
 scos_actions/calibration/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/utils.py b/scos_actions/calibration/utils.py
index 9370c068..fa977653 100644
--- a/scos_actions/calibration/utils.py
+++ b/scos_actions/calibration/utils.py
@@ -70,7 +70,7 @@ def filter_by_parameter(calibrations: dict, value: Union[float, int, bool]) -> d
         msg = (
             f"Could not locate calibration data at {value}"
             + f"\nAttempted lookup using key '{str(value).lower()}'"
-            + f"{f'and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
+            + f"{f' and {float(value)}' if isinstance(value, float) and value.is_integer() else ''}"
             + f"\nUsing calibration data: {calibrations}"
         )
         raise CalibrationEntryMissingException(msg)

From 28ec1b0a8e4e7ac74caf85d70930f7ed3a55a1f3 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:47:49 -0500
Subject: [PATCH 42/62] fix calibration from_json unit tests

---
 scos_actions/calibration/tests/test_calibration.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py
index 297808ef..20c160b8 100644
--- a/scos_actions/calibration/tests/test_calibration.py
+++ b/scos_actions/calibration/tests/test_calibration.py
@@ -90,9 +90,9 @@ def test_to_and_from_json(self, tmp_path: Path):
         self.sample_cal.to_json()
         self.sample_default_cal.to_json()
         # Then load and compare
-        assert self.sample_cal == Calibration.from_json(self.dummy_file_path, False)
+        assert self.sample_cal == Calibration.from_json(self.dummy_file_path)
         assert self.sample_default_cal == Calibration.from_json(
-            self.dummy_default_file_path, True
+            self.dummy_default_file_path
         )
 
         # from_json should ignore extra keys in the loaded file, but not fail
@@ -101,14 +101,13 @@ def test_to_and_from_json(self, tmp_path: Path):
             self.sample_cal.calibration_parameters,
             self.sample_cal.calibration_data,
             "testing",
-            False,
             tmp_path / "testing.json",
             "dt_str",
             [],
             "uid",
         )
         sensor_cal.to_json()
-        loaded_cal = Calibration.from_json(tmp_path / "testing.json", False)
+        loaded_cal = Calibration.from_json(tmp_path / "testing.json")
         loaded_cal.file_path = self.sample_cal.file_path  # Force these to be the same
         assert loaded_cal == self.sample_cal
 
@@ -118,7 +117,7 @@ def test_to_and_from_json(self, tmp_path: Path):
         with open(tmp_path / "almost_a_cal.json", "w") as outfile:
             outfile.write(json.dumps(almost_a_cal))
         with pytest.raises(Exception):
-            almost = Calibration.from_json(tmp_path / "almost_a_cal.json", False)
+            almost = Calibration.from_json(tmp_path / "almost_a_cal.json")
 
     def test_update_not_implemented(self):
         """Ensure the update abstract method is not implemented in the base class"""

From 20845a51395ac59c19fb79be07ec2114420dae16 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:48:17 -0500
Subject: [PATCH 43/62] fix kwargs in call to parent class init

---
 scos_actions/hardware/mocks/mock_sensor.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scos_actions/hardware/mocks/mock_sensor.py b/scos_actions/hardware/mocks/mock_sensor.py
index 4e432f9f..529919de 100644
--- a/scos_actions/hardware/mocks/mock_sensor.py
+++ b/scos_actions/hardware/mocks/mock_sensor.py
@@ -40,14 +40,14 @@ def __init__(
                 "Calibration object provided to mock sensor will not be used to query calibration data."
             )
         super().__init__(
-            signal_analyzer,
-            gps,
-            preselector,
-            switches,
-            location,
-            capabilities,
-            sensor_cal,
-            differential_cal,
+            signal_analyzer=signal_analyzer,
+            gps=gps,
+            preselector=preselector,
+            switches=switches,
+            location=location,
+            capabilities=capabilities,
+            sensor_cal=sensor_cal,
+            differential_cal=differential_cal,
         )
 
     @property

From 3fe4adc7f1a03dfbc3cfffa6997b4b0ac6c55b88 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:49:59 -0500
Subject: [PATCH 44/62] fix incorrect attribute name in unit test

---
 scos_actions/hardware/tests/test_sensor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scos_actions/hardware/tests/test_sensor.py b/scos_actions/hardware/tests/test_sensor.py
index e8f74552..1c3eac95 100644
--- a/scos_actions/hardware/tests/test_sensor.py
+++ b/scos_actions/hardware/tests/test_sensor.py
@@ -28,8 +28,8 @@ def test_mock_sensor_defaults(mock_sensor):
     assert mock_sensor.switches == {}
     assert mock_sensor.location == _mock_location
     assert mock_sensor.capabilities == _mock_capabilities
-    assert mock_sensor.sensor_cal is None
-    assert mock_sensor.differential_cal is None
+    assert mock_sensor.sensor_calibration is None
+    assert mock_sensor.differential_calibration is None
     assert mock_sensor.has_configurable_preselector is False
     assert mock_sensor.has_configurable_preselector is False
     assert mock_sensor.sensor_calibration_data == _mock_sensor_cal_data

From f901514a68cec56ebe4d33ccda9be8aba34145b7 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:54:03 -0500
Subject: [PATCH 45/62] remove tests for no longer used sigan attribute

---
 .../tests/test_acquire_single_freq_fft.py     |  9 ---------
 .../tests/test_single_freq_tdomain_iq.py      |  9 ---------
 .../tests/test_stepped_freq_tdomain_iq.py     | 20 -------------------
 3 files changed, 38 deletions(-)

diff --git a/scos_actions/actions/tests/test_acquire_single_freq_fft.py b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
index 1e65abec..e4d9c9fb 100644
--- a/scos_actions/actions/tests/test_acquire_single_freq_fft.py
+++ b/scos_actions/actions/tests/test_acquire_single_freq_fft.py
@@ -82,12 +82,3 @@ def callback(sender, **kwargs):
             ]
         ]
     )
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_m4s_action"]
-    assert action.description
-    action(
-        sensor=MockSensor(), schedule_entry=SINGLE_FREQUENCY_FFT_ACQUISITION, task_id=1
-    )
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
index 2095d856..774ce91e 100644
--- a/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_single_freq_tdomain_iq.py
@@ -66,12 +66,3 @@ def test_required_components():
     with pytest.raises(RuntimeError):
         action(sensor, SINGLE_TIMEDOMAIN_IQ_ACQUISITION, 1)
     mock_sigan._is_available = True
-
-
-def test_num_samples_skip():
-    action = actions["test_single_frequency_iq_action"]
-    assert action.description
-    action(
-        sensor=MockSensor(), schedule_entry=SINGLE_TIMEDOMAIN_IQ_ACQUISITION, task_id=1
-    )
-    assert action.sensor.signal_analyzer._num_samples_skip == action.parameters["nskip"]
diff --git a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
index 20873922..2183d0bf 100644
--- a/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/tests/test_stepped_freq_tdomain_iq.py
@@ -41,23 +41,3 @@ def callback(sender, **kwargs):
         assert _task_ids[i] == 1
         assert _recording_ids[i] == i + 1
     assert _count == 10
-
-
-def test_num_samples_skip():
-    action = actions["test_multi_frequency_iq_action"]
-    assert action.description
-    action(
-        sensor=MockSensor(),
-        schedule_entry=SINGLE_TIMEDOMAIN_IQ_MULTI_RECORDING_ACQUISITION,
-        task_id=1,
-    )
-    if isinstance(action.parameters["nskip"], list):
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"][-1]
-        )
-    else:
-        assert (
-            action.sensor.signal_analyzer._num_samples_skip
-            == action.parameters["nskip"]
-        )

From ab2831d589432d54035009ede705bede0fef2a3a Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:54:36 -0500
Subject: [PATCH 46/62] avoid error due to debug messages when run with no
 calibration

---
 scos_actions/hardware/sensor.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py
index 3e7552c0..08dbcd9c 100644
--- a/scos_actions/hardware/sensor.py
+++ b/scos_actions/hardware/sensor.py
@@ -280,10 +280,14 @@ def acquire_time_domain_samples(
         logger.debug("***********************************\n")
         logger.debug("Sensor.acquire_time_domain_samples starting")
         logger.debug(f"Number of retries = {retries}")
-        logger.debug(
-            f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
-        )
-        logger.debug(f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}")
+        if self.differential_calibration is not None:
+            logger.debug(
+                f"USING DIFF. CAL: {self.differential_calibration.calibration_data}"
+            )
+        if self.sensor_calibration is not None:
+            logger.debug(
+                f"USING SENSOR CAL: {self.sensor_calibration.calibration_data}"
+            )
         logger.debug("*************************************\n")
 
         max_retries = retries

From 71dcd8501952712d8917ed366c0c4a377f696725 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 11:55:52 -0500
Subject: [PATCH 47/62] fix generator return type hint

---
 scos_actions/actions/acquire_sea_data_product.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index fcadb459..b50ab53b 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -34,6 +34,9 @@
 from environs import Env
 from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
+
+from scos_actions import __version__ as SCOS_ACTIONS_VERSION
+from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.hardware.utils import (
@@ -73,9 +76,6 @@
 from scos_actions.signals import measurement_action_completed, trigger_api_restart
 from scos_actions.utils import convert_datetime_to_millisecond_iso_format, get_days_up
 
-from scos_actions import __version__ as SCOS_ACTIONS_VERSION
-from scos_actions import utils
-
 env = Env()
 logger = logging.getLogger(__name__)
 
@@ -425,7 +425,7 @@ def __init__(self, params: dict, iir_sos: np.ndarray):
         ]
         del params
 
-    def run(self, iqdata: np.ndarray) -> list:
+    def run(self, iqdata: np.ndarray):
         """
         Filter the input IQ data and concurrently compute FFT, PVT, PFP, and APD results.
 

From 4f33c3e1725fc43a7088fff20ed11aec0fabaebd Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Wed, 6 Mar 2024 12:41:24 -0500
Subject: [PATCH 48/62] don't require datetime in sensor cal data

---
 scos_actions/actions/acquire_single_freq_fft.py        | 6 +++---
 scos_actions/actions/acquire_single_freq_tdomain_iq.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/scos_actions/actions/acquire_single_freq_fft.py b/scos_actions/actions/acquire_single_freq_fft.py
index 3ae3fe7a..25f0b3f3 100644
--- a/scos_actions/actions/acquire_single_freq_fft.py
+++ b/scos_actions/actions/acquire_single_freq_fft.py
@@ -173,9 +173,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         # Save measurement results
         measurement_result["data"] = m4s_result
         measurement_result.update(self.parameters)
-        measurement_result["calibration_datetime"] = (
-            self.sensor.sensor_calibration_data["datetime"]
-        )
+        # measurement_result["calibration_datetime"] = (
+        #     self.sensor.sensor_calibration_data["datetime"]
+        # )
         measurement_result["task_id"] = task_id
         measurement_result["classification"] = self.classification
 
diff --git a/scos_actions/actions/acquire_single_freq_tdomain_iq.py b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
index 14796a05..8ddaa11a 100644
--- a/scos_actions/actions/acquire_single_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_single_freq_tdomain_iq.py
@@ -90,9 +90,9 @@ def execute(self, schedule_entry: dict, task_id: int) -> dict:
         measurement_result.update(self.parameters)
         measurement_result["end_time"] = end_time
         measurement_result["task_id"] = task_id
-        measurement_result["calibration_datetime"] = (
-            self.sensor.sensor_calibration_data["datetime"]
-        )
+        # measurement_result["calibration_datetime"] = (
+        #     self.sensor.sensor_calibration_data["datetime"]
+        # )
         measurement_result["classification"] = self.classification
         sigan_settings = self.get_sigan_settings(measurement_result)
         logger.debug(f"sigan settings:{sigan_settings}")

From 1c6f700f2cd659cf0bccd3339dfd78a66a8ba736 Mon Sep 17 00:00:00 2001
From: Anthony Romaniello <aromaniello@ntia.gov>
Date: Tue, 12 Mar 2024 15:13:05 -0400
Subject: [PATCH 49/62] Fill metadata reference from loaded calibration

---
 .../actions/acquire_sea_data_product.py       | 26 ++++++++++++-------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index b50ab53b..9e4e827f 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -112,7 +112,6 @@
 FFT_WINDOW = get_fft_window(FFT_WINDOW_TYPE, FFT_SIZE)
 FFT_WINDOW_ECF = get_fft_window_correction(FFT_WINDOW, "energy")
 IMPEDANCE_OHMS = 50.0
-DATA_REFERENCE_POINT = "noise source output"  # TODO delete
 NUM_ACTORS = 3  # Number of ray actors to initialize
 
 # Create power detectors
@@ -521,7 +520,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             self.iteration_params,
         )
         self.create_global_sensor_metadata(self.sensor)
-        self.create_global_data_product_metadata()
 
         # Initialize remote supervisor actors for IQ processing
         tic = perf_counter()
@@ -534,7 +532,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         logger.debug(f"Spawned {NUM_ACTORS} supervisor actors in {toc-tic:.2f} s")
 
         # Collect all IQ data and spawn data product computation processes
-        dp_procs, cpu_speed = [], []
+        dp_procs, cpu_speed, reference_points = [], [], []
         capture_tic = perf_counter()
         for i, parameters in enumerate(self.iteration_params):
             measurement_result = self.capture_iq(parameters)
@@ -548,16 +546,23 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             toc = perf_counter()
             logger.debug(f"IQ data delivered for processing in {toc-tic:.2f} s")
             # Create capture segment with channel-specific metadata before sigan is reconfigured
-            tic = perf_counter()
             self.create_capture_segment(i, measurement_result)
-            toc = perf_counter()
-            logger.debug(f"Created capture metadata in {toc-tic:.2f} s")
+            # Query CPU speed for later averaging in diagnostics metadata
             cpu_speed.append(get_current_cpu_clock_speed())
+            # Append list of data reference points; later we require these to be identical
+            reference_points.append(measurement_result["reference"])
         capture_toc = perf_counter()
         logger.debug(
             f"Collected all IQ data and started all processing in {capture_toc-capture_tic:.2f} s"
         )
 
+        # Create data product metadata: requires all data reference points
+        # to be identical.
+        assert (
+            len(set(reference_points)) == 1
+        ), "Channel data were scaled to different reference points. Cannot build metadata."
+        self.create_global_data_product_metadata(reference_points[0])
+
         # Collect processed data product results
         all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = (
             [],
@@ -971,7 +976,7 @@ def test_required_components(self):
             trigger_api_restart.send(sender=self.__class__)
         return None
 
-    def create_global_data_product_metadata(self) -> None:
+    def create_global_data_product_metadata(self, data_products_reference: str) -> None:
         p = self.parameters
         num_iq_samples = int(p[SAMPLE_RATE] * p[DURATION_MS] * 1e-3)
         iir_obj = ntia_algorithm.DigitalFilter(
@@ -1016,7 +1021,7 @@ def create_global_data_product_metadata(self) -> None:
             x_step=[p[SAMPLE_RATE] / FFT_SIZE],
             y_units="dBm/Hz",
             processing=[dft_obj.id],
-            reference=DATA_REFERENCE_POINT,  # TODO update
+            reference=data_products_reference,
             description=(
                 "Results of statistical detectors (max, mean, median, 25th_percentile, 75th_percentile, "
                 + "90th_percentile, 95th_percentile, 99th_percentile, 99.9th_percentile, 99.99th_percentile) "
@@ -1036,7 +1041,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pvt_x_axis__s[-1]],
             x_step=[pvt_x_axis__s[1] - pvt_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,  # TODO update
+            reference=data_products_reference,
             description=(
                 "Max- and mean-detected channel power vs. time, with "
                 + f"an integration time of {p[TD_BIN_SIZE_MS]} ms. "
@@ -1063,7 +1068,7 @@ def create_global_data_product_metadata(self) -> None:
             x_stop=[pfp_x_axis__s[-1]],
             x_step=[pfp_x_axis__s[1] - pfp_x_axis__s[0]],
             y_units="dBm",
-            reference=DATA_REFERENCE_POINT,  # TODO update
+            reference=data_products_reference,
             description=(
                 "Channelized periodic frame power statistics reported over"
                 + f" a {p[PFP_FRAME_PERIOD_MS]} ms frame period, with frame resolution"
@@ -1086,6 +1091,7 @@ def create_global_data_product_metadata(self) -> None:
             y_start=[apd_y_axis__dBm[0]],
             y_stop=[apd_y_axis__dBm[-1]],
             y_step=[apd_y_axis__dBm[1] - apd_y_axis__dBm[0]],
+            reference=data_products_reference,
             description=(
                 f"Estimate of the APD, using a {p[APD_BIN_SIZE_DB]} dB "
                 + "bin size for amplitude values. The data payload includes"

From e849a0df4c5dddc57a2b25648f17c5d29f0aaf7b Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 09:35:47 -0600
Subject: [PATCH 50/62] Correct ONBOARD_CALIBRATION_FILE var.

---
 scos_actions/actions/calibrate_y_factor.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index a1593b12..f3a75f4d 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -19,7 +19,7 @@
 r"""Perform a Y-Factor Calibration.
 Supports calculation of gain and noise figure for one or more channels using the
 Y-Factor method. Results are written to the file specified by the environment
-variable ``ONBOARD_SENSOR_CALIBRATION_FILE``. If the sensor already has a sensor calibration
+variable ``ONBOARD_CALIBRATION_FILE``. If the sensor already has a sensor calibration
 object, it is used as the starting point, and copied to a new onboard calibration object
 which is updated by this action. The sensor object's sensor calibration will be set to
 the updated onboard calibration object after this action is run.
@@ -228,13 +228,13 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 calibration_parameters=cal_params,
                 calibration_data=cal_data,
                 calibration_reference=onboard_cal_reference,
-                file_path=env("ONBOARD_SENSOR_CALIBRATION_FILE"),
+                file_path=env("ONBOARD_CALIBRATION_FILE"),
                 last_calibration_datetime=last_cal_datetime,
                 clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
                 sensor_uid=sensor_uid,
             )
         elif self.sensor.sensor_calibration.file_path == env(
-            "ONBOARD_SENSOR_CALIBRATION_FILE"
+            "ONBOARD_CALIBRATION_FILE"
         ):
             # Already using an onboard cal file.
             logger.debug("Onboard calibration file already in use. Continuing.")

From b17b749847c43f4d807732bea879d20c16889a05 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 09:43:07 -0600
Subject: [PATCH 51/62] Use Path for file_path.

---
 scos_actions/actions/calibrate_y_factor.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py
index f3a75f4d..af00dcbe 100644
--- a/scos_actions/actions/calibrate_y_factor.py
+++ b/scos_actions/actions/calibrate_y_factor.py
@@ -77,6 +77,7 @@
 import logging
 import os
 import time
+from pathlib import Path
 
 import numpy as np
 from environs import Env
@@ -228,7 +229,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
                 calibration_parameters=cal_params,
                 calibration_data=cal_data,
                 calibration_reference=onboard_cal_reference,
-                file_path=env("ONBOARD_CALIBRATION_FILE"),
+                file_path=Path(env("ONBOARD_CALIBRATION_FILE")),
                 last_calibration_datetime=last_cal_datetime,
                 clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate,
                 sensor_uid=sensor_uid,

From 4e22e836edce827b6a3c4c4445e6c241dd17169d Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 10:02:10 -0600
Subject: [PATCH 52/62] debug

---
 scos_actions/actions/acquire_sea_data_product.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 9e4e827f..0ba93d54 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -451,6 +451,7 @@ class NasctnSeaDataProduct(Action):
     def __init__(self, parameters: dict):
         super().__init__(parameters)
         # Assume preselector is present
+        self.total_channel_data_length = None
         rf_path_name = utils.get_parameter(RF_PATH, self.parameters)
         self.rf_path = {self.PRESELECTOR_PATH_KEY: rf_path_name}
 
@@ -1110,6 +1111,7 @@ def create_global_data_product_metadata(self, data_products_reference: str) -> N
             + pfp_length * len(PFP_M3_DETECTOR) * 2
             + apd_graph.length
         )
+        logger.debug(f"Total channel length:{self.total_channel_data_length}")
 
     def create_capture_segment(
         self,

From d9d9bb4646698855ad4d0e7249589c423195a187 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 11:23:34 -0600
Subject: [PATCH 53/62] create global data product metadata after first IQ
 capture so reference is available and total_channel_data_length is known
 during capture creation.

---
 scos_actions/actions/acquire_sea_data_product.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 0ba93d54..2e438dd4 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -521,7 +521,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             self.iteration_params,
         )
         self.create_global_sensor_metadata(self.sensor)
-
         # Initialize remote supervisor actors for IQ processing
         tic = perf_counter()
         # This uses iteration_params[0] because
@@ -535,8 +534,11 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         # Collect all IQ data and spawn data product computation processes
         dp_procs, cpu_speed, reference_points = [], [], []
         capture_tic = perf_counter()
+
         for i, parameters in enumerate(self.iteration_params):
             measurement_result = self.capture_iq(parameters)
+            if i == 0:
+                self.create_global_data_product_metadata(measurement_result["reference"])
             # Start data product processing but do not block next IQ capture
             tic = perf_counter()
 
@@ -562,7 +564,7 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         assert (
             len(set(reference_points)) == 1
         ), "Channel data were scaled to different reference points. Cannot build metadata."
-        self.create_global_data_product_metadata(reference_points[0])
+
 
         # Collect processed data product results
         all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = (

From 8012c2b92a4e9a016074c77d03ba168175208811 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 13:00:05 -0600
Subject: [PATCH 54/62] Add expired method to SensorCalibration.

---
 .../calibration/sensor_calibration.py         | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index a4f23a54..b12a4287 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -3,10 +3,13 @@
 """
 import logging
 from dataclasses import dataclass
+from datetime import datetime
+from environs import Env
 from typing import Dict, List, Union
 
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.utils import CalibrationEntryMissingException
+from scos_actions.utils import parse_datetime_iso_format_str
 
 logger = logging.getLogger(__name__)
 
@@ -84,3 +87,21 @@ def update(
 
         # Write updated calibration data to file
         self.to_json()
+
+    def expired(self) -> bool:
+        env = Env()
+        time_limit = env("CALIBRATION_EXPIRATION_LIMIT")
+        if time_limit is None:
+            return False
+        elif self.calibration_data is None:
+            return True;
+        elif len(self.calibration_data) == 0:
+            return True;
+        else:
+            now = datetime.now()
+            for cal_data in self.calibration_data:
+                cal_datetime =  parse_datetime_iso_format_str(cal_data["datetime"])
+                elapsed = now - cal_datetime
+                if elapsed.total_seconds() > time_limit:
+                    return True
+            return False
\ No newline at end of file

From 9a1004eaa5ed588451e0e2879f7c4098d0c64b23 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 13:18:11 -0600
Subject: [PATCH 55/62] set CALIBRATION_EXPIRATION_LIMIT to None if not set.

---
 scos_actions/calibration/sensor_calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index b12a4287..950f0b35 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -90,7 +90,7 @@ def update(
 
     def expired(self) -> bool:
         env = Env()
-        time_limit = env("CALIBRATION_EXPIRATION_LIMIT")
+        time_limit = env("CALIBRATION_EXPIRATION_LIMIT", default=None)
         if time_limit is None:
             return False
         elif self.calibration_data is None:

From bdb142bf59c51d042637158ca411a0ad250f2e6f Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 14:49:24 -0600
Subject: [PATCH 56/62] recusive check if calibration has expired and
 associated tests.

---
 .../calibration/sensor_calibration.py         | 28 ++++++++++++++-----
 .../tests/test_sensor_calibration.py          | 25 +++++++++++++++++
 2 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 950f0b35..23dccf90 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -98,10 +98,24 @@ def expired(self) -> bool:
         elif len(self.calibration_data) == 0:
             return True;
         else:
-            now = datetime.now()
-            for cal_data in self.calibration_data:
-                cal_datetime =  parse_datetime_iso_format_str(cal_data["datetime"])
-                elapsed = now - cal_datetime
-                if elapsed.total_seconds() > time_limit:
-                    return True
-            return False
\ No newline at end of file
+            now_string = datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
+            now = parse_datetime_iso_format_str(now_string)
+            cal_data = self.calibration_data
+            return has_expired_cal_data(cal_data, now,time_limit)
+
+def has_expired_cal_data( cal_data: dict, now: datetime, time_limit: int) -> bool:
+    expired = False
+    if "datetime" in cal_data:
+        expired = expired or date_expired(cal_data, now, time_limit)
+
+    for key, value in cal_data.items():
+        if isinstance(value, dict):
+            expired = expired or has_expired_cal_data(value,now,time_limit)
+    return expired
+
+def date_expired( cal_data: dict, now: datetime, time_limit: int):
+    cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
+    elapsed = now - cal_datetime
+    if elapsed.total_seconds() > time_limit:
+        return True
+    return False
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 743faf1b..1f351e41 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -13,6 +13,7 @@
 
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.sensor_calibration import SensorCalibration
+from scos_actions.calibration.sensor_calibration import has_expired_cal_data
 from scos_actions.calibration.tests.utils import recursive_check_keys
 from scos_actions.calibration.utils import CalibrationException
 from scos_actions.tests.resources.utils import easy_gain
@@ -331,3 +332,27 @@ def test_update(self):
         assert cal.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0
         assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0
+
+    def test_has_expired_cal_data_not_expired(self):
+        cal_date = "2024-03-14T15:48:38.039Z"
+        now_date = "2024-03-14T15:49:38.039Z"
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
+        expired = has_expired_cal_data(cal_data,  parse_datetime_iso_format_str( now_date), 100)
+        assert expired == False
+
+    def test_has_expired_cal_data_expired(self):
+        cal_date = "2024-03-14T15:48:38.039Z"
+        now_date = "2024-03-14T15:49:38.039Z"
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
+        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 30)
+        assert expired == True
+
+    def test_has_expired_cal_data_multipledates_expired(self):
+        cal_date_1 = "2024-03-14T15:48:38.039Z"
+        cal_date_2 = "2024-03-14T15:40:38.039Z"
+        now_date = "2024-03-14T15:49:38.039Z"
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_1},}, "true":{ "datetime": cal_date_2},}}
+        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
+        assert expired == True
+        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_2},}, "true":{ "datetime": cal_date_1},}}
+        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
\ No newline at end of file

From 16366c78e4a41dacdd0fc243181f54bf7e3a10f5 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 14:51:14 -0600
Subject: [PATCH 57/62] pre-commit

---
 sample_debug.py                               |  1 +
 .../actions/acquire_sea_data_product.py       |  5 +-
 .../acquire_stepped_freq_tdomain_iq.py        |  4 +-
 .../actions/interfaces/measurement_action.py  |  1 +
 .../calibration/sensor_calibration.py         | 18 +++---
 .../tests/test_sensor_calibration.py          | 56 +++++++++++++++----
 scos_actions/hardware/sigan_iface.py          |  1 +
 scos_actions/hardware/tests/test_sigan.py     |  1 +
 scos_actions/metadata/structs/capture.py      |  1 +
 scos_actions/signal_processing/calibration.py |  1 +
 .../tests/test_calibration.py                 |  1 +
 .../signal_processing/tests/test_fft.py       |  1 +
 .../signal_processing/tests/test_filtering.py |  1 +
 .../tests/test_power_analysis.py              |  1 +
 .../tests/test_unit_conversion.py             |  1 +
 15 files changed, 73 insertions(+), 21 deletions(-)

diff --git a/sample_debug.py b/sample_debug.py
index cc04ae29..3cf415fe 100644
--- a/sample_debug.py
+++ b/sample_debug.py
@@ -2,6 +2,7 @@
 This is a sample file showing how an action be created and called for debugging purposes
 using a mock signal analyzer.
 """
+
 import json
 
 from scos_actions.actions.acquire_single_freq_fft import SingleFrequencyFftAcquisition
diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 2e438dd4..4046fc33 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -538,7 +538,9 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         for i, parameters in enumerate(self.iteration_params):
             measurement_result = self.capture_iq(parameters)
             if i == 0:
-                self.create_global_data_product_metadata(measurement_result["reference"])
+                self.create_global_data_product_metadata(
+                    measurement_result["reference"]
+                )
             # Start data product processing but do not block next IQ capture
             tic = perf_counter()
 
@@ -565,7 +567,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
             len(set(reference_points)) == 1
         ), "Channel data were scaled to different reference points. Cannot build metadata."
 
-
         # Collect processed data product results
         all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = (
             [],
diff --git a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
index 7278e5f1..6de6c5a9 100644
--- a/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
+++ b/scos_actions/actions/acquire_stepped_freq_tdomain_iq.py
@@ -38,6 +38,8 @@
 import logging
 
 import numpy as np
+
+from scos_actions import utils
 from scos_actions.actions.acquire_single_freq_tdomain_iq import (
     CAL_ADJUST,
     DURATION_MS,
@@ -51,8 +53,6 @@
 from scos_actions.signals import measurement_action_completed
 from scos_actions.utils import get_parameter
 
-from scos_actions import utils
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py
index fe5e197c..c8fb1847 100644
--- a/scos_actions/actions/interfaces/measurement_action.py
+++ b/scos_actions/actions/interfaces/measurement_action.py
@@ -3,6 +3,7 @@
 from typing import Optional
 
 import numpy as np
+
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.hardware.sensor import Sensor
 from scos_actions.metadata.structs import ntia_sensor
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 23dccf90..f5e03b63 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -1,12 +1,14 @@
 """
 TODO
 """
+
 import logging
 from dataclasses import dataclass
 from datetime import datetime
-from environs import Env
 from typing import Dict, List, Union
 
+from environs import Env
+
 from scos_actions.calibration.interfaces.calibration import Calibration
 from scos_actions.calibration.utils import CalibrationEntryMissingException
 from scos_actions.utils import parse_datetime_iso_format_str
@@ -94,26 +96,28 @@ def expired(self) -> bool:
         if time_limit is None:
             return False
         elif self.calibration_data is None:
-            return True;
+            return True
         elif len(self.calibration_data) == 0:
-            return True;
+            return True
         else:
             now_string = datetime.utcnow().isoformat(timespec="milliseconds") + "Z"
             now = parse_datetime_iso_format_str(now_string)
             cal_data = self.calibration_data
-            return has_expired_cal_data(cal_data, now,time_limit)
+            return has_expired_cal_data(cal_data, now, time_limit)
+
 
-def has_expired_cal_data( cal_data: dict, now: datetime, time_limit: int) -> bool:
+def has_expired_cal_data(cal_data: dict, now: datetime, time_limit: int) -> bool:
     expired = False
     if "datetime" in cal_data:
         expired = expired or date_expired(cal_data, now, time_limit)
 
     for key, value in cal_data.items():
         if isinstance(value, dict):
-            expired = expired or has_expired_cal_data(value,now,time_limit)
+            expired = expired or has_expired_cal_data(value, now, time_limit)
     return expired
 
-def date_expired( cal_data: dict, now: datetime, time_limit: int):
+
+def date_expired(cal_data: dict, now: datetime, time_limit: int):
     cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
     elapsed = now - cal_datetime
     if elapsed.total_seconds() > time_limit:
diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py
index 1f351e41..01d70854 100644
--- a/scos_actions/calibration/tests/test_sensor_calibration.py
+++ b/scos_actions/calibration/tests/test_sensor_calibration.py
@@ -12,8 +12,10 @@
 import pytest
 
 from scos_actions.calibration.interfaces.calibration import Calibration
-from scos_actions.calibration.sensor_calibration import SensorCalibration
-from scos_actions.calibration.sensor_calibration import has_expired_cal_data
+from scos_actions.calibration.sensor_calibration import (
+    SensorCalibration,
+    has_expired_cal_data,
+)
 from scos_actions.calibration.tests.utils import recursive_check_keys
 from scos_actions.calibration.utils import CalibrationException
 from scos_actions.tests.resources.utils import easy_gain
@@ -336,23 +338,57 @@ def test_update(self):
     def test_has_expired_cal_data_not_expired(self):
         cal_date = "2024-03-14T15:48:38.039Z"
         now_date = "2024-03-14T15:49:38.039Z"
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
-        expired = has_expired_cal_data(cal_data,  parse_datetime_iso_format_str( now_date), 100)
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date},
+                },
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 100
+        )
         assert expired == False
 
     def test_has_expired_cal_data_expired(self):
         cal_date = "2024-03-14T15:48:38.039Z"
         now_date = "2024-03-14T15:49:38.039Z"
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date},},}}
-        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 30)
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date},
+                },
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 30
+        )
         assert expired == True
 
     def test_has_expired_cal_data_multipledates_expired(self):
         cal_date_1 = "2024-03-14T15:48:38.039Z"
         cal_date_2 = "2024-03-14T15:40:38.039Z"
         now_date = "2024-03-14T15:49:38.039Z"
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_1},}, "true":{ "datetime": cal_date_2},}}
-        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date_1},
+                },
+                "true": {"datetime": cal_date_2},
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 100
+        )
         assert expired == True
-        cal_data = {"3550": {"20.0": {"false": {"datetime": cal_date_2},}, "true":{ "datetime": cal_date_1},}}
-        expired = has_expired_cal_data(cal_data, parse_datetime_iso_format_str( now_date), 100)
\ No newline at end of file
+        cal_data = {
+            "3550": {
+                "20.0": {
+                    "false": {"datetime": cal_date_2},
+                },
+                "true": {"datetime": cal_date_1},
+            }
+        }
+        expired = has_expired_cal_data(
+            cal_data, parse_datetime_iso_format_str(now_date), 100
+        )
diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py
index cb0e85cb..8b6d207c 100644
--- a/scos_actions/hardware/sigan_iface.py
+++ b/scos_actions/hardware/sigan_iface.py
@@ -4,6 +4,7 @@
 from typing import Dict, Optional
 
 from its_preselector.web_relay import WebRelay
+
 from scos_actions.hardware.utils import power_cycle_sigan
 
 logger = logging.getLogger(__name__)
diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py
index d4434d78..4fce6562 100644
--- a/scos_actions/hardware/tests/test_sigan.py
+++ b/scos_actions/hardware/tests/test_sigan.py
@@ -1,4 +1,5 @@
 import pytest
+
 from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer
 
 
diff --git a/scos_actions/metadata/structs/capture.py b/scos_actions/metadata/structs/capture.py
index 59fe6128..abcff4fb 100644
--- a/scos_actions/metadata/structs/capture.py
+++ b/scos_actions/metadata/structs/capture.py
@@ -1,6 +1,7 @@
 from typing import Optional
 
 import msgspec
+
 from scos_actions.metadata.structs.ntia_sensor import Calibration, SiganSettings
 from scos_actions.metadata.utils import SIGMF_OBJECT_KWARGS
 
diff --git a/scos_actions/signal_processing/calibration.py b/scos_actions/signal_processing/calibration.py
index 3f98ade8..e1a4c847 100644
--- a/scos_actions/signal_processing/calibration.py
+++ b/scos_actions/signal_processing/calibration.py
@@ -5,6 +5,7 @@
 from its_preselector.preselector import Preselector
 from numpy.typing import NDArray
 from scipy.constants import Boltzmann
+
 from scos_actions.calibration.utils import CalibrationException
 from scos_actions.signal_processing.unit_conversion import (
     convert_celsius_to_fahrenheit,
diff --git a/scos_actions/signal_processing/tests/test_calibration.py b/scos_actions/signal_processing/tests/test_calibration.py
index c47a2a37..6b29b472 100644
--- a/scos_actions/signal_processing/tests/test_calibration.py
+++ b/scos_actions/signal_processing/tests/test_calibration.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.calibration
 """
+
 import numpy as np
 from scipy.constants import Boltzmann
 
diff --git a/scos_actions/signal_processing/tests/test_fft.py b/scos_actions/signal_processing/tests/test_fft.py
index a848340f..1ea7458c 100644
--- a/scos_actions/signal_processing/tests/test_fft.py
+++ b/scos_actions/signal_processing/tests/test_fft.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.fft
 """
+
 import numpy as np
 import pytest
 from scipy.signal import get_window
diff --git a/scos_actions/signal_processing/tests/test_filtering.py b/scos_actions/signal_processing/tests/test_filtering.py
index d33c37c6..0df7b845 100644
--- a/scos_actions/signal_processing/tests/test_filtering.py
+++ b/scos_actions/signal_processing/tests/test_filtering.py
@@ -5,6 +5,7 @@
 tests mostly exist to ensure that tests will fail if substantial changes
 are made to the wrappers.
 """
+
 import numpy as np
 import pytest
 from scipy.signal import ellip, ellipord, firwin, kaiserord, sos2zpk, sosfreqz
diff --git a/scos_actions/signal_processing/tests/test_power_analysis.py b/scos_actions/signal_processing/tests/test_power_analysis.py
index 42e2db75..02aaf454 100644
--- a/scos_actions/signal_processing/tests/test_power_analysis.py
+++ b/scos_actions/signal_processing/tests/test_power_analysis.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.power_analysis
 """
+
 from enum import EnumMeta
 
 import numpy as np
diff --git a/scos_actions/signal_processing/tests/test_unit_conversion.py b/scos_actions/signal_processing/tests/test_unit_conversion.py
index 4da61f86..fe404472 100644
--- a/scos_actions/signal_processing/tests/test_unit_conversion.py
+++ b/scos_actions/signal_processing/tests/test_unit_conversion.py
@@ -1,6 +1,7 @@
 """
 Unit test for scos_actions.signal_processing.unit_conversion
 """
+
 import numpy as np
 import pytest
 

From 15d63d87b92af7a8018386e7ac525e7869cef55c Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 14:56:41 -0600
Subject: [PATCH 58/62] get CALIBRATION_EXPIRATION_LIMIT as int.

---
 scos_actions/calibration/sensor_calibration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index f5e03b63..7ccdb03f 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -92,7 +92,7 @@ def update(
 
     def expired(self) -> bool:
         env = Env()
-        time_limit = env("CALIBRATION_EXPIRATION_LIMIT", default=None)
+        time_limit = env.int("CALIBRATION_EXPIRATION_LIMIT", default=None)
         if time_limit is None:
             return False
         elif self.calibration_data is None:

From 57452a6b7d00527701827e88e373c37f5909bc8b Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 16:05:15 -0600
Subject: [PATCH 59/62] move import of ray into __call__ just in case.

---
 scos_actions/actions/acquire_sea_data_product.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 560375cf..03e3f764 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -30,7 +30,6 @@
 
 import numpy as np
 import psutil
-import ray
 from environs import Env
 from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
@@ -506,6 +505,8 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         action_start_tic = perf_counter()
         # Ray should have already been initialized within scos-sensor,
         # but check and initialize just in case.
+        import ray
+
         if not ray.is_initialized():
             logger.info("Initializing ray.")
             logger.info("Set RAY_INIT=true to avoid initializing within " + __name__)

From 521518ab91a7bf8dfa9d8bf0b71be0f49171a92c Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 16:09:21 -0600
Subject: [PATCH 60/62] Restore import or ray.

---
 scos_actions/actions/acquire_sea_data_product.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py
index 03e3f764..bd2a8d82 100644
--- a/scos_actions/actions/acquire_sea_data_product.py
+++ b/scos_actions/actions/acquire_sea_data_product.py
@@ -23,6 +23,7 @@
 import logging
 import lzma
 import platform
+import ray
 import sys
 from enum import EnumMeta
 from time import perf_counter
@@ -505,7 +506,6 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
         action_start_tic = perf_counter()
         # Ray should have already been initialized within scos-sensor,
         # but check and initialize just in case.
-        import ray
 
         if not ray.is_initialized():
             logger.info("Initializing ray.")

From 23faff1da732ca7ad35da96fdea73d2d6ccf2e94 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Thu, 14 Mar 2024 16:29:07 -0600
Subject: [PATCH 61/62] Add logging.

---
 scos_actions/calibration/sensor_calibration.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index 7ccdb03f..f2f66718 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -93,6 +93,7 @@ def update(
     def expired(self) -> bool:
         env = Env()
         time_limit = env.int("CALIBRATION_EXPIRATION_LIMIT", default=None)
+        logger.debug("Checking if calibration has expired.")
         if time_limit is None:
             return False
         elif self.calibration_data is None:
@@ -121,5 +122,8 @@ def date_expired(cal_data: dict, now: datetime, time_limit: int):
     cal_datetime = parse_datetime_iso_format_str(cal_data["datetime"])
     elapsed = now - cal_datetime
     if elapsed.total_seconds() > time_limit:
+        logger.debug(
+            f"Calibration {cal_data} has expired at {elapsed.total_seconds()} seconds old."
+        )
         return True
     return False

From 838b9507557775f12a813cfe5d857cdce3150bc9 Mon Sep 17 00:00:00 2001
From: Doug Boulware <dboulware@ntia.doc.gov>
Date: Fri, 15 Mar 2024 09:04:34 -0600
Subject: [PATCH 62/62] Additional debug logging.

---
 scos_actions/calibration/interfaces/calibration.py | 1 +
 scos_actions/calibration/sensor_calibration.py     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/scos_actions/calibration/interfaces/calibration.py b/scos_actions/calibration/interfaces/calibration.py
index b243bea4..011e56b5 100644
--- a/scos_actions/calibration/interfaces/calibration.py
+++ b/scos_actions/calibration/interfaces/calibration.py
@@ -129,3 +129,4 @@ def to_json(self) -> None:
         dict_to_json.pop("file_path", None)
         with open(self.file_path, "w") as outfile:
             outfile.write(json.dumps(dict_to_json))
+        logger.debug("Finished updating calibration file.")
diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py
index f2f66718..0330b944 100644
--- a/scos_actions/calibration/sensor_calibration.py
+++ b/scos_actions/calibration/sensor_calibration.py
@@ -54,6 +54,7 @@ def update(
         :param temp_degC: Temperature at calibration time, in degrees Celsius.
         :param file_path: File path for saving the updated calibration data.
         """
+        logger.debug("Updating calibration file.")
         try:
             # Get existing calibration data entry which will be updated
             data_entry = self.get_calibration_dict(params)