From 668d429c9c3eb9611c2f9bfcad868f09bc787626 Mon Sep 17 00:00:00 2001 From: dboulware Date: Fri, 26 Apr 2024 11:21:45 -0600 Subject: [PATCH] Revert "Revert "Sea 178 test actions loading"" --- scos_actions/__init__.py | 2 +- scos_actions/actions/calibrate_y_factor.py | 2 -- scos_actions/actions/interfaces/action.py | 19 ++++++----- .../actions/interfaces/measurement_action.py | 7 ++-- scos_actions/actions/logger.py | 34 ------------------- scos_actions/actions/sync_gps.py | 4 +-- .../calibration/sensor_calibration.py | 8 ----- .../calibration/tests/test_calibration.py | 1 - .../tests/test_sensor_calibration.py | 22 ------------ ...yml => test_SEA_CBRS_Measure_Baseline.yml} | 14 +++----- .../test_multi_frequency_iq_action.yml | 1 + .../test_single_frequency_iq_action.yml | 1 + .../test_single_frequency_m4s_action.yml | 1 + .../configs/actions/test_survey_iq_action.yml | 1 + scos_actions/discover/__init__.py | 33 ++++++++++-------- scos_actions/hardware/gps_iface.py | 6 ++-- scos_actions/hardware/mocks/mock_gps.py | 5 +-- scos_actions/hardware/mocks/mock_sigan.py | 6 ++++ scos_actions/hardware/sensor.py | 31 ++++++++++++++++- scos_actions/hardware/sigan_iface.py | 6 ++++ scos_actions/settings.py | 14 +++++++- 21 files changed, 106 insertions(+), 112 deletions(-) delete mode 100644 scos_actions/actions/logger.py rename scos_actions/configs/actions/{test_nasctn_sea_data_product.yml => test_SEA_CBRS_Measure_Baseline.yml} (74%) diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index 6dcf770e..9158871f 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -1 +1 @@ -__version__ = "9.0.0" +__version__ = "10.0.0" diff --git a/scos_actions/actions/calibrate_y_factor.py b/scos_actions/actions/calibrate_y_factor.py index 69fc5d5b..636596f0 100644 --- a/scos_actions/actions/calibrate_y_factor.py +++ b/scos_actions/actions/calibrate_y_factor.py @@ -231,14 +231,12 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int): logger.debug(f"cal_params: {cal_params}") cal_data = dict() last_cal_datetime = get_datetime_str_now() - clock_rate_lookup_by_sample_rate = [] self.sensor.sensor_calibration = SensorCalibration( calibration_parameters=cal_params, calibration_data=cal_data, calibration_reference=onboard_cal_reference, file_path=Path(env("ONBOARD_CALIBRATION_FILE")), last_calibration_datetime=last_cal_datetime, - clock_rate_lookup_by_sample_rate=clock_rate_lookup_by_sample_rate, sensor_uid=sensor_uid, ) elif self.sensor.sensor_calibration.file_path == env( diff --git a/scos_actions/actions/interfaces/action.py b/scos_actions/actions/interfaces/action.py index 16a51fc9..cee001ae 100644 --- a/scos_actions/actions/interfaces/action.py +++ b/scos_actions/actions/interfaces/action.py @@ -59,15 +59,16 @@ def configure_sigan(self, params: dict): def configure_preselector(self, params: dict): preselector = self.sensor.preselector - if self.PRESELECTOR_PATH_KEY in params: - path = params[self.PRESELECTOR_PATH_KEY] - logger.debug(f"Setting preselector RF path: {path}") - preselector.set_state(path) - elif self.sensor.has_configurable_preselector: - # Require the RF path to be specified if the sensor has a preselector. - raise ParameterException( - f"No {self.PRESELECTOR_PATH_KEY} value specified in the YAML config." - ) + if self.sensor.has_configurable_preselector: + if self.PRESELECTOR_PATH_KEY in params: + path = params[self.PRESELECTOR_PATH_KEY] + logger.debug(f"Setting preselector RF path: {path}") + preselector.set_state(path) + else: + # Require the RF path to be specified if the sensor has a preselector. + raise ParameterException( + f"No {self.PRESELECTOR_PATH_KEY} value specified in the YAML config." + ) else: # No preselector in use, so do not require an RF path pass diff --git a/scos_actions/actions/interfaces/measurement_action.py b/scos_actions/actions/interfaces/measurement_action.py index e7ad088a..1400937a 100644 --- a/scos_actions/actions/interfaces/measurement_action.py +++ b/scos_actions/actions/interfaces/measurement_action.py @@ -67,15 +67,16 @@ def get_calibration(self, measurement_result: dict) -> ntia_sensor.Calibration: noise_figure=round( measurement_result["applied_calibration"]["noise_figure"], 3 ), - temperature=round( - self.sensor.sensor_calibration_data["temperature"], 1 - ), reference=measurement_result["reference"], ) if "compression_point" in measurement_result["applied_calibration"]: cal_meta.compression_point = measurement_result["applied_calibration"][ "compression_point" ] + if "temperature" in self.sensor.sensor_calibration_data: + cal_meta.temperature = round( + self.sensor.sensor_calibration_data["temperature"], 1 + ) return cal_meta def create_metadata( diff --git a/scos_actions/actions/logger.py b/scos_actions/actions/logger.py deleted file mode 100644 index ae2c6e60..00000000 --- a/scos_actions/actions/logger.py +++ /dev/null @@ -1,34 +0,0 @@ -"""A simple example action that logs a message.""" - -import logging -from typing import Optional - -from scos_actions.actions.interfaces.action import Action -from scos_actions.hardware.sensor import Sensor - -logger = logging.getLogger(__name__) - -LOGLVL_INFO = 20 -LOGLVL_ERROR = 40 - - -class Logger(Action): - """Log the message "running test {name}/{tid}". - - This is useful for testing and debugging. - - `{name}` will be replaced with the parent schedule entry's name, and - `{tid}` will be replaced with the sequential task id. - - """ - - def __init__(self, loglvl=LOGLVL_INFO): - super().__init__(parameters={"name": "logger"}) - self.loglvl = loglvl - - def __call__(self, sensor: Optional[Sensor], schedule_entry: dict, task_id: int): - msg = "running test {name}/{tid}" - schedule_entry_name = schedule_entry["name"] - logger.log( - level=self.loglvl, msg=msg.format(name=schedule_entry_name, tid=task_id) - ) diff --git a/scos_actions/actions/sync_gps.py b/scos_actions/actions/sync_gps.py index b0f88543..7163a12d 100644 --- a/scos_actions/actions/sync_gps.py +++ b/scos_actions/actions/sync_gps.py @@ -19,12 +19,12 @@ def __init__(self, parameters: dict): def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int): logger.debug("Syncing to GPS") self.sensor = sensor - dt = self.sensor.gps.get_gps_time() + dt = self.sensor.gps.get_gps_time(self.sensor) date_cmd = ["date", "-s", "{:}".format(dt.strftime("%Y/%m/%d %H:%M:%S"))] subprocess.check_output(date_cmd, shell=True) logger.info(f"Set system time to GPS time {dt.ctime()}") - location = sensor.gps.get_location() + location = sensor.gps.get_location(self.sensor) if location is None: raise RuntimeError("Unable to synchronize to GPS") diff --git a/scos_actions/calibration/sensor_calibration.py b/scos_actions/calibration/sensor_calibration.py index 1ce985ee..e5c94759 100644 --- a/scos_actions/calibration/sensor_calibration.py +++ b/scos_actions/calibration/sensor_calibration.py @@ -25,16 +25,8 @@ class provides an implementation for the update method to allow calibration """ last_calibration_datetime: str - clock_rate_lookup_by_sample_rate: List[Dict[str, float]] sensor_uid: str - def get_clock_rate(self, sample_rate: Union[float, int]) -> Union[float, int]: - """Find the clock rate (Hz) using the given sample_rate (samples per second)""" - for mapping in self.clock_rate_lookup_by_sample_rate: - if mapping["sample_rate"] == sample_rate: - return mapping["clock_frequency"] - return sample_rate - def update( self, params: dict, diff --git a/scos_actions/calibration/tests/test_calibration.py b/scos_actions/calibration/tests/test_calibration.py index 20c160b8..9c74d9a1 100644 --- a/scos_actions/calibration/tests/test_calibration.py +++ b/scos_actions/calibration/tests/test_calibration.py @@ -103,7 +103,6 @@ def test_to_and_from_json(self, tmp_path: Path): "testing", tmp_path / "testing.json", "dt_str", - [], "uid", ) sensor_cal.to_json() diff --git a/scos_actions/calibration/tests/test_sensor_calibration.py b/scos_actions/calibration/tests/test_sensor_calibration.py index bd4c610c..f5e9f581 100644 --- a/scos_actions/calibration/tests/test_sensor_calibration.py +++ b/scos_actions/calibration/tests/test_sensor_calibration.py @@ -83,17 +83,6 @@ def setup_calibration_file(self, tmp_path: Path): cal_data["sensor_uid"] = "SAMPLE_CALIBRATION" cal_data["calibration_reference"] = "TESTING" - # Add SR/CF lookup table - cal_data["clock_rate_lookup_by_sample_rate"] = [] - for sr in self.sample_rates: - cr = sr - while cr <= 40e6: - cr *= 2 - cr /= 2 - cal_data["clock_rate_lookup_by_sample_rate"].append( - {"sample_rate": int(sr), "clock_frequency": int(cr)} - ) - # Create the JSON architecture for the calibration data cal_data["calibration_data"] = {} cal_data["calibration_parameters"] = ["sample_rate", "frequency", "gain"] @@ -151,7 +140,6 @@ def test_sensor_calibration_dataclass_fields(self): # Note: does not check field order assert fields == { "last_calibration_datetime": str, - "clock_rate_lookup_by_sample_rate": List[Dict[str, float]], "sensor_uid": str, } @@ -167,13 +155,6 @@ def test_field_validator(self): [], {}, False, Path(""), datetime.datetime.now(), [], "uid" ) - def test_get_clock_rate(self): - """Test the get_clock_rate method""" - # Test getting a clock rate by sample rate - assert self.sample_cal.get_clock_rate(10e6) == 40e6 - # If there isn't an entry, the sample rate should be returned - assert self.sample_cal.get_clock_rate(-999) == -999 - def test_get_calibration_dict_exact_match_lookup(self): calibration_datetime = get_datetime_str_now() calibration_params = ["sample_rate", "frequency"] @@ -187,7 +168,6 @@ def test_get_calibration_dict_exact_match_lookup(self): calibration_reference="testing", file_path=Path(""), last_calibration_datetime=calibration_datetime, - clock_rate_lookup_by_sample_rate=[], sensor_uid="TESTING", ) cal_data = cal.get_calibration_dict({"sample_rate": 100.0, "frequency": 200.0}) @@ -206,7 +186,6 @@ def test_get_calibration_dict_within_range(self): calibration_reference="testing", file_path=Path("test_calibration.json"), last_calibration_datetime=calibration_datetime, - clock_rate_lookup_by_sample_rate=[], sensor_uid="TESTING", ) @@ -234,7 +213,6 @@ def test_update(self): calibration_reference="testing", file_path=test_cal_path, last_calibration_datetime=calibration_datetime, - clock_rate_lookup_by_sample_rate=[], sensor_uid="TESTING", ) action_params = {"sample_rate": 100.0, "frequency": 200.0} diff --git a/scos_actions/configs/actions/test_nasctn_sea_data_product.yml b/scos_actions/configs/actions/test_SEA_CBRS_Measure_Baseline.yml similarity index 74% rename from scos_actions/configs/actions/test_nasctn_sea_data_product.yml rename to scos_actions/configs/actions/test_SEA_CBRS_Measure_Baseline.yml index fca3a521..8fc5d5c6 100644 --- a/scos_actions/configs/actions/test_nasctn_sea_data_product.yml +++ b/scos_actions/configs/actions/test_SEA_CBRS_Measure_Baseline.yml @@ -1,26 +1,22 @@ nasctn_sea_data_product: - name: test_nasctn_sea_data_product + name: test_SEA_CBRS_Measure_Baseline rf_path: antenna + calibration_adjust: False # IIR filter settings - iir_apply: True iir_gpass_dB: 0.1 # Max passband ripple below unity gain iir_gstop_dB: 40 # Minimum stopband attenuation iir_pb_edge_Hz: 5e6 # Passband edge frequency iir_sb_edge_Hz: 5.008e6 # Stopband edge frequency -# Mean/Max FFT settings - fft_size: 175 +# FFT settings nffts: 320e3 - fft_window_type: flattop # See scipy.signal.get_window for supported input # PFP frame pfp_frame_period_ms: 10 # APD downsampling settings - apd_bin_size_dB: 0.5 # Set to 0 or negative for no downsampling - apd_min_bin_dBm: -180 + apd_bin_size_dB: 1.0 # Set to 0 or negative for no downsampling apd_max_bin_dBm: -30 + apd_min_bin_dBm: -180 # Time domain power statistics settings td_bin_size_ms: 10 -# Round all power results to X decimal places - round_to_places: 2 # Sigan Settings preamp_enable: True reference_level: -25 diff --git a/scos_actions/configs/actions/test_multi_frequency_iq_action.yml b/scos_actions/configs/actions/test_multi_frequency_iq_action.yml index 8074c921..b1d2a1ff 100644 --- a/scos_actions/configs/actions/test_multi_frequency_iq_action.yml +++ b/scos_actions/configs/actions/test_multi_frequency_iq_action.yml @@ -17,3 +17,4 @@ stepped_frequency_time_domain_iq: nskip: 15.36e4 calibration_adjust: False classification: UNCLASSIFIED + rf_path: antenna diff --git a/scos_actions/configs/actions/test_single_frequency_iq_action.yml b/scos_actions/configs/actions/test_single_frequency_iq_action.yml index 15908352..d8fa80f9 100644 --- a/scos_actions/configs/actions/test_single_frequency_iq_action.yml +++ b/scos_actions/configs/actions/test_single_frequency_iq_action.yml @@ -7,3 +7,4 @@ single_frequency_time_domain_iq: nskip: 15.36e4 calibration_adjust: False classification: UNCLASSIFIED + rf_path: antenna diff --git a/scos_actions/configs/actions/test_single_frequency_m4s_action.yml b/scos_actions/configs/actions/test_single_frequency_m4s_action.yml index 3220bf4d..f20312f6 100644 --- a/scos_actions/configs/actions/test_single_frequency_m4s_action.yml +++ b/scos_actions/configs/actions/test_single_frequency_m4s_action.yml @@ -8,3 +8,4 @@ single_frequency_fft: nskip: 15.36e4 calibration_adjust: False classification: UNCLASSIFIED + rf_path: antenna diff --git a/scos_actions/configs/actions/test_survey_iq_action.yml b/scos_actions/configs/actions/test_survey_iq_action.yml index f64540e1..fe16e7c6 100644 --- a/scos_actions/configs/actions/test_survey_iq_action.yml +++ b/scos_actions/configs/actions/test_survey_iq_action.yml @@ -27,3 +27,4 @@ stepped_frequency_time_domain_iq: - 10000 nskip: 15.36e4 calibration_adjust: False + rf_path: antenna diff --git a/scos_actions/discover/__init__.py b/scos_actions/discover/__init__.py index 0fa0128f..30e1cb29 100644 --- a/scos_actions/discover/__init__.py +++ b/scos_actions/discover/__init__.py @@ -1,20 +1,11 @@ from scos_actions.actions import action_classes -from scos_actions.actions.logger import Logger from scos_actions.actions.monitor_sigan import MonitorSignalAnalyzer from scos_actions.actions.sync_gps import SyncGps from scos_actions.discover.yaml import load_from_yaml -from scos_actions.settings import ACTION_DEFINITIONS_DIR +from scos_actions.settings import ACTION_DEFINITIONS_DIR, SIGAN_CLASS, SIGAN_MODULE -actions = { - "logger": Logger(), -} -test_actions = { - "test_sync_gps": SyncGps(parameters={"name": "test_sync_gps"}), - "test_monitor_sigan": MonitorSignalAnalyzer( - parameters={"name": "test_monitor_sigan"} - ), - "logger": Logger(), -} +actions = {} +test_actions = {} def init( @@ -31,6 +22,18 @@ def init( return yaml_actions, yaml_test_actions -yaml_actions, yaml_test_actions = init() -actions.update(yaml_actions) -test_actions.update(yaml_test_actions) +if ( + SIGAN_MODULE == "scos_actions.hardware.mocks.mock_sigan" + and SIGAN_CLASS == "MockSignalAnalyzer" +): + yaml_actions, yaml_test_actions = init() + actions.update(yaml_actions) + test_actions.update( + { + "test_sync_gps": SyncGps(parameters={"name": "test_sync_gps"}), + "test_monitor_sigan": MonitorSignalAnalyzer( + parameters={"name": "test_monitor_sigan"} + ), + } + ) + test_actions.update(yaml_test_actions) diff --git a/scos_actions/hardware/gps_iface.py b/scos_actions/hardware/gps_iface.py index 1b9da39b..ac220ef7 100644 --- a/scos_actions/hardware/gps_iface.py +++ b/scos_actions/hardware/gps_iface.py @@ -3,9 +3,11 @@ class GPSInterface(ABC): @abstractmethod - def get_location(self, timeout_s=1): + def get_location( + self, sensor: "scos_actions.hardware.sensor.Sensor", timeout_s: float = 1 + ): pass @abstractmethod - def get_gps_time(self): + def get_gps_time(self, sensor: "scos_actions.hardware.sensor.Sensor"): pass diff --git a/scos_actions/hardware/mocks/mock_gps.py b/scos_actions/hardware/mocks/mock_gps.py index 4c05a6ff..769a7de1 100644 --- a/scos_actions/hardware/mocks/mock_gps.py +++ b/scos_actions/hardware/mocks/mock_gps.py @@ -7,10 +7,11 @@ class MockGPS(GPSInterface): - def get_location(timeout_s=1): + + def get_location(self, sensor, timeout_s=1): logger.warning("Using mock GPS!") return 39.995118, -105.261572, 1651.0 - def get_gps_time(self): + def get_gps_time(self, sensor): logger.warning("Using mock GPS!") return datetime.now() diff --git a/scos_actions/hardware/mocks/mock_sigan.py b/scos_actions/hardware/mocks/mock_sigan.py index b4394e06..bb8d6a9f 100644 --- a/scos_actions/hardware/mocks/mock_sigan.py +++ b/scos_actions/hardware/mocks/mock_sigan.py @@ -6,6 +6,7 @@ import numpy as np +from scos_actions import __package__ as SCOS_ACTIONS_NAME from scos_actions import __version__ as SCOS_ACTIONS_VERSION from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface from scos_actions.utils import get_datetime_str_now @@ -42,6 +43,7 @@ def __init__( self._reference_level = -30 self._is_available = True self._plugin_version = SCOS_ACTIONS_VERSION + self._plugin_name = SCOS_ACTIONS_NAME self._firmware_version = "1.2.3" self._api_version = "v1.2.3" @@ -60,6 +62,10 @@ def is_available(self): def plugin_version(self): return self._plugin_version + @property + def plugin_name(self): + return self._plugin_name + @property def sample_rate(self): return self._sample_rate diff --git a/scos_actions/hardware/sensor.py b/scos_actions/hardware/sensor.py index 47198b93..6a9e9701 100644 --- a/scos_actions/hardware/sensor.py +++ b/scos_actions/hardware/sensor.py @@ -4,11 +4,12 @@ import logging from typing import Any, Dict, List, Optional +import numpy as np +from numpy.typing import ArrayLike from its_preselector.preselector import Preselector from its_preselector.web_relay import WebRelay from scos_actions.calibration.differential_calibration import DifferentialCalibration -from scos_actions.calibration.interfaces.calibration import Calibration from scos_actions.calibration.sensor_calibration import SensorCalibration from scos_actions.hardware.gps_iface import GPSInterface from scos_actions.hardware.sigan_iface import SignalAnalyzerInterface @@ -246,6 +247,24 @@ def recompute_calibration_data(self, params: dict) -> None: if not recomputed: logger.warning("Failed to recompute calibration data") + def check_sensor_overload(self, data: ArrayLike) -> bool: + """Check for sensor overload in the measurement data.""" + # explicitly check is not None since 1db compression could be 0 + if self.sensor_calibration_data["compression_point"] is not None: + time_domain_avg_power = 10 * np.log10(np.mean(np.abs(data) ** 2)) + time_domain_avg_power += ( + 10 * np.log10(1 / (2 * 50)) + 30 + ) # Convert log(V^2) to dBm + return bool( + time_domain_avg_power + > self.sensor_calibration_data["compression_point"] + ) + else: + logger.debug( + "Compression point is None, returning False for sensor overload." + ) + return False + def acquire_time_domain_samples( self, num_samples: int, @@ -288,6 +307,7 @@ def acquire_time_domain_samples( logger.debug("*************************************\n") max_retries = retries + sensor_overload = False # Acquire samples from signal analyzer if self.signal_analyzer is not None: while True: @@ -363,6 +383,15 @@ def acquire_time_domain_samples( measurement_result["applied_calibration"]["compression_point"] = ( self.sensor_calibration_data["compression_point"] ) + sensor_overload = self.check_sensor_overload( + measurement_result["data"] + ) + if sensor_overload: + logger.warning("Sensor overload occurred!") + # measurement_result["overload"] could be true based on sigan overload or sensor overload + measurement_result["overload"] = ( + measurement_result["overload"] or sensor_overload + ) applied_cal = measurement_result["applied_calibration"] logger.debug(f"Setting applied_calibration to: {applied_cal}") else: diff --git a/scos_actions/hardware/sigan_iface.py b/scos_actions/hardware/sigan_iface.py index bbf42de6..5f0df68f 100644 --- a/scos_actions/hardware/sigan_iface.py +++ b/scos_actions/hardware/sigan_iface.py @@ -32,6 +32,12 @@ def plugin_version(self) -> str: """Returns the version of the SCOS plugin defining this interface.""" pass + @property + @abstractmethod + def plugin_name(self) -> str: + """Returns the name of the SCOS plugin defining this interface.""" + pass + @property def firmware_version(self) -> str: """Returns the version of the signal analyzer firmware.""" diff --git a/scos_actions/settings.py b/scos_actions/settings.py index 7d45d179..eeca61ad 100644 --- a/scos_actions/settings.py +++ b/scos_actions/settings.py @@ -1,4 +1,6 @@ import logging +import sys +from os import path from pathlib import Path from environs import Env @@ -21,11 +23,21 @@ logger.debug(f"scos-actions: MOCK_SIGAN:{MOCK_SIGAN}") MOCK_SIGAN_RANDOM = env.bool("MOCK_SIGAN_RANDOM", default=False) logger.debug(f"scos-actions: MOCK_SIGAN_RANDOM:{MOCK_SIGAN_RANDOM}") -RUNNING_TESTS = env.bool("RUNNING_TESTS", False) +__cmd = path.split(sys.argv[0])[-1] +RUNNING_TESTS = env.bool("RUNNING_TESTS", "test" in __cmd) logger.debug(f"scos-actions: RUNNING_TESTS:{RUNNING_TESTS}") logger.debug(f"scos-actions: RUNNING_TESTS:{RUNNING_TESTS}") FQDN = env("FQDN", None) logger.debug(f"scos-actions: FQDN:{FQDN}") + +SIGAN_MODULE = env.str("SIGAN_MODULE", default=None) +if RUNNING_TESTS: + SIGAN_MODULE = "scos_actions.hardware.mocks.mock_sigan" +logger.debug(f"scos-actions: SIGAN_MODULE:{SIGAN_MODULE}") +SIGAN_CLASS = env.str("SIGAN_CLASS", default=None) +if RUNNING_TESTS: + SIGAN_CLASS = "MockSignalAnalyzer" +logger.debug(f"scos-actions: SIGAN_CLASS:{SIGAN_CLASS}") SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) logger.debug(f"scos-actions: SIGAN_POWER_SWITCH:{SIGAN_POWER_SWITCH}") SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None)