diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index 87c6f8ed..54ccb5d2 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -1,2 +1 @@ __version__ = "7.1.0" - diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 11ced67c..bc975e65 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -78,7 +78,7 @@ from scos_actions import utils from scos_actions.actions.interfaces.action import Action -from scos_actions.calibration import sensor_calibration +from scos_actions.calibration import sensor_calibration, default_sensor_calibration from scos_actions.hardware.mocks.mock_gps import MockGPS from scos_actions.hardware.sigan_iface import SIGAN_SETTINGS_KEYS from scos_actions.settings import SENSOR_CALIBRATION_FILE @@ -253,7 +253,7 @@ def calibrate(self, params): assert ( sample_rate == noise_off_measurement_result["sample_rate"] ), "Sample rate mismatch" - + sigan_params = {k: v for k, v in params.items() if k in SIGAN_SETTINGS_KEYS} # Apply IIR filtering to both captures if configured if self.iir_apply: # Estimate of IIR filter ENBW does NOT account for passband ripple in sensor transfer function! @@ -262,9 +262,13 @@ def calibrate(self, params): noise_on_data = sosfilt(self.iir_sos, noise_on_measurement_result["data"]) noise_off_data = sosfilt(self.iir_sos, noise_off_measurement_result["data"]) else: + if default_sensor_calibration: + raise Exception( + "Calibrations without IIR filter cannot be performed with default calibration." + ) + logger.debug("Skipping IIR filtering") # Get ENBW from sensor calibration - sigan_params = {k: v for k, v in params.items() if k in SIGAN_SETTINGS_KEYS} assert set(sensor_calibration.calibration_parameters) <= set( sigan_params.keys() ), f"Action parameters do not include all required calibration parameters" @@ -289,7 +293,7 @@ def calibrate(self, params): # Update sensor calibration with results sensor_calibration.update( - params, + sigan_params, utils.get_datetime_str_now(), gain, noise_figure, diff --git a/scos_actions/calibration/__init__.py b/scos_actions/calibration/__init__.py index 14326964..e78f4c7e 100644 --- a/scos_actions/calibration/__init__.py +++ b/scos_actions/calibration/__init__.py @@ -1,13 +1,17 @@ import logging -from pathlib import Path +from os import path from scos_actions.calibration.calibration import Calibration, load_from_json -from scos_actions.settings import SENSOR_CALIBRATION_FILE, SIGAN_CALIBRATION_FILE +from scos_actions.settings import ( + DEFAULT_CALIBRATION_FILE, + SENSOR_CALIBRATION_FILE, + SIGAN_CALIBRATION_FILE, +) logger = logging.getLogger(__name__) -def get_sigan_calibration(sigan_cal_file: Path) -> Calibration: +def get_sigan_calibration(sigan_cal_file: str) -> Calibration: """ Load signal analyzer calibration data from file. @@ -23,7 +27,7 @@ def get_sigan_calibration(sigan_cal_file: Path) -> Calibration: return sigan_cal -def get_sensor_calibration(sensor_cal_file: Path) -> Calibration: +def get_sensor_calibration(sensor_cal_file: str) -> Calibration: """ Load sensor calibration data from file. @@ -39,9 +43,48 @@ def get_sensor_calibration(sensor_cal_file: Path) -> Calibration: return sensor_cal -logger.debug(f"Loading sensor cal file: {SENSOR_CALIBRATION_FILE}") -sensor_calibration = get_sensor_calibration(SENSOR_CALIBRATION_FILE) -logger.debug(f"Loading sigan cal file: {SIGAN_CALIBRATION_FILE}") -sigan_calibration = get_sigan_calibration(SIGAN_CALIBRATION_FILE) +def check_for_default_calibration(cal_file_path: str, cal_type: str) -> bool: + default_cal = False + if cal_file_path == DEFAULT_CALIBRATION_FILE: + default_cal = True + logger.warning( + f"***************LOADING DEFAULT {cal_type} CALIBRATION***************" + ) + return default_cal + + +sensor_calibration = None +if SENSOR_CALIBRATION_FILE is None or SENSOR_CALIBRATION_FILE == "": + logger.warning( + "No sensor calibration file specified. Not loading calibration file." + ) +elif not path.exists(SENSOR_CALIBRATION_FILE): + logger.warning( + SENSOR_CALIBRATION_FILE + + " does not exist. Not loading sensor calibration file." + ) +else: + logger.debug(f"Loading sensor cal file: {SENSOR_CALIBRATION_FILE}") + default_sensor_calibration = check_for_default_calibration( + SENSOR_CALIBRATION_FILE, "Sensor" + ) + sensor_calibration = get_sensor_calibration(SENSOR_CALIBRATION_FILE) + +sigan_calibration = None +default_sensor_calibration = False +default_sigan_calibration = False +if SIGAN_CALIBRATION_FILE is None or SIGAN_CALIBRATION_FILE == "": + logger.warning("No sigan calibration file specified. Not loading calibration file.") +elif not path.exists(SIGAN_CALIBRATION_FILE): + logger.warning( + SIGAN_CALIBRATION_FILE + " does not exist. Not loading sigan calibration file." + ) +else: + logger.debug(f"Loading sigan cal file: {SIGAN_CALIBRATION_FILE}") + default_sigan_calibration = check_for_default_calibration( + SIGAN_CALIBRATION_FILE, "Sigan" + ) + sigan_calibration = get_sigan_calibration(SIGAN_CALIBRATION_FILE) + if sensor_calibration: logger.debug(f"Last sensor cal: {sensor_calibration.last_calibration_datetime}") diff --git a/scos_actions/calibration/calibration.py b/scos_actions/calibration/calibration.py index d028d004..9d724af7 100644 --- a/scos_actions/calibration/calibration.py +++ b/scos_actions/calibration/calibration.py @@ -38,14 +38,14 @@ def get_calibration_dict(self, cal_params: List[Union[float, int, bool]]) -> dic then the input to this method could be ``["15360000.0", "40"]``. :return: The calibration data corresponding to the input parameter values. """ - # Check if the sample rate was calibrated + cal_data = self.calibration_data - # raise Exception(self) for i, setting_value in enumerate(cal_params): setting = self.calibration_parameters[i] logger.debug(f"Looking up calibration for {setting} at {setting_value}") cal_data = filter_by_parameter(cal_data, setting_value) logger.debug(f"Got calibration data: {cal_data}") + return cal_data def update( @@ -77,9 +77,10 @@ def update( """ cal_data = self.calibration_data self.last_calibration_datetime = calibration_datetime_str - + if len(self.calibration_parameters) == 0: + self.calibration_parameters = list(params.keys()) # Ensure all required calibration parameters were used - if not set(params.keys()) >= set(self.calibration_parameters): + elif not set(params.keys()) >= set(self.calibration_parameters): raise Exception( "Not enough parameters specified to update calibration.\n" + f"Required parameters are {self.calibration_parameters}" diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py index 8d07df36..75fc1eb7 100644 --- a/scos_actions/calibration/tests/test_calibration.py +++ b/scos_actions/calibration/tests/test_calibration.py @@ -310,12 +310,6 @@ def test_update(self): assert cal_from_file.calibration_data["100.0"]["200.0"]["gain"] == 30.0 assert cal_from_file.calibration_data["100.0"]["200.0"]["noise_figure"] == 5.0 - def test_default_sensor_cal(self): - assert sensor_calibration is not None - - def test_default_sigan_cal_location(self): - assert sigan_calibration is not None - def test_filter_by_paramter_integer(self): calibrations = {"200.0": {"some_cal_data"}, 300.0: {"more cal data"}} filtered_data = filter_by_parameter(calibrations, 200) diff --git a/scos_actions/configs/sensor_calibration_example.json b/scos_actions/configs/sensor_calibration_example.json deleted file mode 100644 index 669c7445..00000000 --- a/scos_actions/configs/sensor_calibration_example.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "calibration_data": { - "10000000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - }, - "15360000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - } - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [ - "sample_rate", - "frequency", - "gain" - ], - "clock_rate_lookup_by_sample_rate": [ - { - "clock_frequency": 40000000.0, - "sample_rate": 10000000.0 - }, - { - "clock_frequency": 30720000.0, - "sample_rate": 15360000.0 - } - ], - "sensor_uid": "EXAMPLE", - "calibration_reference": "noise source output" -} diff --git a/scos_actions/configs/sigan_calibration_example.json b/scos_actions/configs/sigan_calibration_example.json deleted file mode 100644 index 669c7445..00000000 --- a/scos_actions/configs/sigan_calibration_example.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "calibration_data": { - "10000000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - }, - "15360000.0": { - "650000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - }, - "660000000.0": { - "40.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - }, - "60.0": { - "datetime": "1970-01-01T00:00:00.000000Z", - "gain": 30.0, - "noise_figure": 1.5, - "1db_compression_point": 22.5, - "enbw": 10000000.0, - "temperature": 26.85 - } - } - } - }, - "last_calibration_datetime": "1970-01-01T00:00:00.000000Z", - "calibration_parameters": [ - "sample_rate", - "frequency", - "gain" - ], - "clock_rate_lookup_by_sample_rate": [ - { - "clock_frequency": 40000000.0, - "sample_rate": 10000000.0 - }, - { - "clock_frequency": 30720000.0, - "sample_rate": 15360000.0 - } - ], - "sensor_uid": "EXAMPLE", - "calibration_reference": "noise source output" -} diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py index 29942705..28379ab6 100644 --- a/scos_actions/hardware/mocks/mock_sigan.py +++ b/scos_actions/hardware/mocks/mock_sigan.py @@ -26,6 +26,24 @@ class MockSignalAnalyzer(SignalAnalyzerInterface): def __init__(self, randomize_values=False): super().__init__() + # Define the default calibration dicts + self.DEFAULT_SIGAN_CALIBRATION = { + "datetime": get_datetime_str_now(), + "gain": 0, # Defaults to gain setting + "enbw": None, # Defaults to sample rate + "noise_figure": 0, + "1db_compression_point": 100, + "temperature": 26.85, + } + + self.DEFAULT_SENSOR_CALIBRATION = { + "datetime": get_datetime_str_now(), + "gain": 0, # Defaults to sigan gain + "enbw": None, # Defaults to sigan enbw + "noise_figure": None, # Defaults to sigan noise figure + "1db_compression_point": None, # Defaults to sigan compression + preselector gain + "temperature": 26.85, + } self.auto_dc_offset = False self._frequency = 700e6 self._sample_rate = 10e6 @@ -57,11 +75,11 @@ def is_available(self): @property def plugin_version(self): return self._plugin_version - + @property def firmware_version(self): return self._firmware_version - + @property def api_version(self): return self._api_version @@ -182,3 +200,20 @@ def last_calibration_time(self): def update_calibration(self, params): pass + + def recompute_sensor_calibration_data(self, cal_args: list) -> None: + if self.sensor_calibration is not None: + self.sensor_calibration_data.update( + self.sensor_calibration.get_calibration_dict(cal_args) + ) + else: + logger.warning("Sensor calibration does not exist.") + + def recompute_sigan_calibration_data(self, cal_args: list) -> None: + """Set the sigan calibration data based on the current tuning""" + if self.sigan_calibration is not None: + self.sigan_calibration_data.update( + self.sigan_calibration.get_calibration_dict(cal_args) + ) + else: + logger.warning("Sigan calibration does not exist.") diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index ee6a43e0..786025d3 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -5,14 +5,8 @@ from scos_actions.calibration import sensor_calibration, sigan_calibration from scos_actions.capabilities import capabilities -from scos_actions.hardware.hardware_configuration_exception import ( - HardwareConfigurationException, -) from scos_actions.hardware.utils import power_cycle_sigan -from scos_actions.utils import ( - convert_string_to_millisecond_iso_format, - get_datetime_str_now, -) +from scos_actions.utils import convert_string_to_millisecond_iso_format logger = logging.getLogger(__name__) @@ -30,26 +24,10 @@ class SignalAnalyzerInterface(ABC): def __init__(self): - # Define the default calibration dicts - self.DEFAULT_SIGAN_CALIBRATION = { - "datetime": get_datetime_str_now(), - "gain": None, # Defaults to gain setting - "enbw": None, # Defaults to sample rate - "noise_figure": 0, - "1db_compression_point": 100, - "temperature": 26.85, - } - - self.DEFAULT_SENSOR_CALIBRATION = { - "datetime": get_datetime_str_now(), - "gain": None, # Defaults to sigan gain - "enbw": None, # Defaults to sigan enbw - "noise_figure": None, # Defaults to sigan noise figure - "1db_compression_point": None, # Defaults to sigan compression + preselector gain - "temperature": 26.85, - } - self.sensor_calibration_data = copy.deepcopy(self.DEFAULT_SENSOR_CALIBRATION) - self.sigan_calibration_data = copy.deepcopy(self.DEFAULT_SIGAN_CALIBRATION) + self.sensor_calibration_data = {} + self.sigan_calibration_data = {} + self.sensor_calibration = sensor_calibration + self.sigan_calibration = sigan_calibration @property def last_calibration_time(self) -> str: @@ -135,7 +113,7 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None: logger.info("Attempting to power cycle the signal analyzer and reconnect.") try: power_cycle_sigan() - except HardwareConfigurationException as hce: + except Exception as hce: logger.warning(f"Unable to power cycle sigan: {hce}") return try: @@ -151,19 +129,23 @@ def power_cycle_and_connect(self, sleep_time: float = 2.0) -> None: return def recompute_sensor_calibration_data(self, cal_args: list) -> None: - self.sensor_calibration_data = self.DEFAULT_SENSOR_CALIBRATION.copy() + self.sensor_calibration_data = {} if sensor_calibration is not None: self.sensor_calibration_data.update( sensor_calibration.get_calibration_dict(cal_args) ) + else: + logger.warning("Sensor calibration does not exist.") def recompute_sigan_calibration_data(self, cal_args: list) -> None: + self.sigan_calibration_data = {} """Set the sigan calibration data based on the current tuning""" - self.sigan_calibration_data = self.DEFAULT_SIGAN_CALIBRATION.copy() if sigan_calibration is not None: self.sigan_calibration_data.update( sigan_calibration.get_calibration_dict(cal_args) ) + else: + logger.warning("Sigan calibration does not exist.") def get_status(self): try: diff --git a/scos_actions/hardware/tests/test_sigan.py b/scos_actions/hardware/tests/test_sigan.py new file mode 100644 index 00000000..df980ecf --- /dev/null +++ b/scos_actions/hardware/tests/test_sigan.py @@ -0,0 +1,10 @@ +from environs import Env + +from scos_actions.hardware.mocks.mock_sigan import MockSignalAnalyzer + + +def test_sigan_default_cal(): + sigan = MockSignalAnalyzer() + sigan.recompute_sensor_calibration_data([]) + sensor_cal = sigan.sensor_calibration_data + assert sensor_cal["gain"] == 0 diff --git a/scos_actions/metadata/utils.py b/scos_actions/metadata/utils.py index 6253049a..d814249f 100644 --- a/scos_actions/metadata/utils.py +++ b/scos_actions/metadata/utils.py @@ -33,10 +33,10 @@ def construct_geojson_point( def _enc_hook(obj: Any) -> Any: - #While isinstance is recommended, it was causing a - #Recurrsion error and I don't think we have to worry - #about subytpes here. - if type(obj) == np.float64: + # While isinstance is recommended, it was causing a + # Recurrsion error and I don't think we have to worry + # about subytpes here. + if type(obj) == np.float64: return float(obj) elif type(obj) == np.bool_: return bool(obj) diff --git a/scos_actions/settings.py b/scos_actions/settings.py index e8236f75..00cf702a 100644 --- a/scos_actions/settings.py +++ b/scos_actions/settings.py @@ -1,4 +1,5 @@ import logging +from os import path from pathlib import Path from django.conf import settings @@ -11,21 +12,26 @@ CONFIG_DIR = Path(__file__).parent.resolve() / "configs" ACTION_DEFINITIONS_DIR = CONFIG_DIR / "actions" +if not settings.configured or not hasattr(settings, "DEFAULT_CALIBRATION_FILE"): + DEFAULT_CALIBRATION_FILE = path.join(CONFIG_DIR, "default_calibration.json") +else: + DEFAULT_CALIBRATION_FILE = settings.DEFAULT_CALIBRATION_FILE + # set sigan_calibration file and sensor_calibration_file if not settings.configured or not hasattr(settings, "SIGAN_CALIBRATION_FILE"): - logger.warning("Using default sigan cal file.") - SIGAN_CALIBRATION_FILE = CONFIG_DIR / "sigan_calibration_example.json" + logger.warning("Sigan calibration file is not defined.") + SIGAN_CALIBRATION_FILE = "" sigan_calibration = None else: - SIGAN_CALIBRATION_FILE = Path(settings.SIGAN_CALIBRATION_FILE) + SIGAN_CALIBRATION_FILE = settings.SIGAN_CALIBRATION_FILE logger.debug(f"SCOS_ACTIONS: SIGAN_CALIBRATION_FILE: {SIGAN_CALIBRATION_FILE}") if not settings.configured or not hasattr(settings, "SENSOR_CALIBRATION_FILE"): - logger.warning("Using default sensor cal file.") - SENSOR_CALIBRATION_FILE = CONFIG_DIR / "sensor_calibration_example.json" + logger.warning("Sensor calibration file is not defined.") + SENSOR_CALIBRATION_FILE = "" sensor_calibration = None else: - SENSOR_CALIBRATION_FILE = Path(settings.SENSOR_CALIBRATION_FILE) + SENSOR_CALIBRATION_FILE = settings.SENSOR_CALIBRATION_FILE logger.debug(f"SCOS_ACTIONS: SENSOR_CALIBRATION_FILE: {SENSOR_CALIBRATION_FILE}") SWITCH_CONFIGS_DIR = env("SWITCH_CONFIGS_DIR", default=None)