diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b719edcc..d9d65765 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/asottile/pyupgrade - rev: v3.4.0 + rev: v3.10.1 hooks: - id: pyupgrade args: ["--py38-plus"] @@ -30,12 +30,12 @@ repos: types: [file, python] args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black types: [file, python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.34.0 + rev: v0.35.0 hooks: - id: markdownlint types: [file, markdown] diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index 123dd5c0..9eaeb606 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -1 +1 @@ -__version__ = "6.3.2" +__version__ = "6.3.3" diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py index 746b1654..e9a46d9f 100644 --- a/scos_actions/actions/acquire_sea_data_product.py +++ b/scos_actions/actions/acquire_sea_data_product.py @@ -20,9 +20,10 @@ Currently in development. """ -import gc import logging import lzma +import platform +import sys from enum import EnumMeta from time import perf_counter from typing import Tuple @@ -30,8 +31,11 @@ import numpy as np import psutil import ray +from environs import Env +from its_preselector import __version__ as PRESELECTOR_API_VERSION from scipy.signal import sos2tf, sosfilt +from scos_actions import __version__ as SCOS_ACTIONS_VERSION from scos_actions import utils from scos_actions.actions.interfaces.action import Action from scos_actions.capabilities import SENSOR_DEFINITION_HASH, SENSOR_LOCATION @@ -53,6 +57,7 @@ ntia_sensor, ) from scos_actions.metadata.structs.capture import CaptureSegment +from scos_actions.settings import SCOS_SENSOR_GIT_TAG from scos_actions.signal_processing.apd import get_apd from scos_actions.signal_processing.fft import ( get_fft, @@ -74,6 +79,7 @@ from scos_actions.status import start_time from scos_actions.utils import convert_datetime_to_millisecond_iso_format, get_days_up +env = Env() logger = logging.getLogger(__name__) if not ray.is_initialized(): @@ -271,24 +277,34 @@ def run(self, iq: ray.ObjectRef) -> Tuple[np.ndarray, np.ndarray]: :return: Two NumPy arrays: the first has shape (2, 400) and the second is 1D with length 2. The first array contains the (max, mean) detector results and the second array contains - the (max-of-max, median-of-mean) summary statistics. + the (max-of-max, median-of-mean, mean, median) single-valued + summary statistics. """ # Reshape IQ data into blocks and calculate power n_blocks = len(iq) // self.block_size iq_pwr = calculate_power_watts( iq.reshape((n_blocks, self.block_size)), self.impedance_ohms ) + # Get true median power + pvt_median = np.median(iq_pwr.flatten()) # Apply max/mean detectors pvt_result = apply_statistical_detector(iq_pwr, self.detector, axis=1) - # Get single value median/max statistics - pvt_summary = np.array([pvt_result[0].max(), np.median(pvt_result[1])]) + # Get single value statistics: (max-of-max, median-of-mean, mean, median) + pvt_summary = np.array( + [ + pvt_result[0].max(), + np.median(pvt_result[1]), + pvt_result[1].mean(), + pvt_median, + ] + ) # Convert to dBm and account for RF/baseband power difference # Note: convert_watts_to_dBm is not used to avoid NumExpr usage # for the relatively small arrays pvt_result, pvt_summary = ( 10.0 * np.log10(x) + 27.0 for x in [pvt_result, pvt_summary] ) - # Return order ((max array, mean array), (max-of-max, median-of-mean)) + # Return order ((max array, mean array), (max-of-max, median-of-mean, mean, median)) return pvt_result, pvt_summary @@ -512,6 +528,7 @@ def __call__(self, schedule_entry, task_id): schedule_entry, self.iteration_params, ) + self.create_global_sensor_metadata() self.create_global_data_product_metadata() # Initialize remote supervisor actors for IQ processing @@ -550,7 +567,13 @@ def __call__(self, schedule_entry, task_id): ) # Collect processed data product results - all_data, max_max_ch_pwrs, med_mean_ch_pwrs = [], [], [] + all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = ( + [], + [], + [], + [], + [], + ) result_tic = perf_counter() for i, channel_data_process in enumerate(dp_procs): # Retrieve object references for channel data @@ -564,6 +587,8 @@ def __call__(self, schedule_entry, task_id): data, summaries = data # Split the tuple max_max_ch_pwrs.append(DATA_TYPE(summaries[0])) med_mean_ch_pwrs.append(DATA_TYPE(summaries[1])) + mean_ch_pwrs.append(DATA_TYPE(summaries[2])) + median_ch_pwrs.append(DATA_TYPE(summaries[3])) del summaries if i == 3: # Separate condition is intentional # APD result: append instead of extend, @@ -585,6 +610,8 @@ def __call__(self, schedule_entry, task_id): all_data = self.compress_bytes_data(np.array(all_data).tobytes()) self.sigmf_builder.set_max_of_max_channel_powers(max_max_ch_pwrs) self.sigmf_builder.set_median_of_mean_channel_powers(med_mean_ch_pwrs) + self.sigmf_builder.set_mean_channel_powers(mean_ch_pwrs) + self.sigmf_builder.set_median_channel_powers(median_ch_pwrs) # Get diagnostics last to record action runtime self.capture_diagnostics( action_start_tic, cpu_speed @@ -645,6 +672,9 @@ def capture_diagnostics(self, action_start_tic: float, cpu_speeds: list) -> None CPU temperature, CPU overheating status, CPU uptime, SCOS start time, and SCOS uptime. + Software versions: the OS platform, Python version, scos_actions + version, and preselector API version. + The total action runtime is also recorded. Preselector X410 Setup requires: @@ -670,6 +700,8 @@ def capture_diagnostics(self, action_start_tic: float, cpu_speeds: list) -> None :param action_start_tic: Action start timestamp, as would be returned by ``time.perf_counter()`` + :param cpu_speeds: List of CPU speed values, recorded at + consecutive points as the action has been running. """ tic = perf_counter() # Read SPU sensors @@ -758,6 +790,15 @@ def capture_diagnostics(self, action_start_tic: float, cpu_speeds: list) -> None except: logger.warning("Failed to get SSD SMART data") + # Get software versions + software_diag = { + "system_platform": platform.platform(), + "python_version": sys.version.split()[0], + "scos_sensor_version": SCOS_SENSOR_GIT_TAG, + "scos_actions_version": SCOS_ACTIONS_VERSION, + "preselector_api_version": PRESELECTOR_API_VERSION, + } + toc = perf_counter() logger.debug(f"Got all diagnostics in {toc-tic} s") diagnostics = { @@ -765,12 +806,43 @@ def capture_diagnostics(self, action_start_tic: float, cpu_speeds: list) -> None "preselector": ntia_diagnostics.Preselector(**ps_diag), "spu": ntia_diagnostics.SPU(**spu_diag), "computer": ntia_diagnostics.Computer(**cpu_diag), + "software": ntia_diagnostics.Software(**software_diag), "action_runtime": round(perf_counter() - action_start_tic, 2), } # Add diagnostics to SigMF global object self.sigmf_builder.set_diagnostics(ntia_diagnostics.Diagnostics(**diagnostics)) + def create_global_sensor_metadata(self): + # Add (minimal) ntia-sensor metadata to the sigmf_builder: + # sensor ID, serial numbers for preselector, sigan, and computer + # overall sensor_spec version, e.g. "Prototype Rev. 3" + # sensor definition hash, to link to full sensor definition + self.sigmf_builder.set_sensor( + ntia_sensor.Sensor( + sensor_spec=ntia_core.HardwareSpec( + id=self.sensor_definition["sensor_spec"]["id"], + version=self.sensor_definition["sensor_spec"]["version"], + ), + preselector=ntia_sensor.Preselector( + preselector_spec=ntia_core.HardwareSpec( + id=self.sensor_definition["preselector"]["preselector_spec"][ + "id" + ] + ) + ), + signal_analyzer=ntia_sensor.SignalAnalyzer( + sigan_spec=ntia_core.HardwareSpec( + id=self.sensor_definition["signal_analyzer"]["sigan_spec"]["id"] + ) + ), + computer_spec=ntia_sensor.HardwareSpec( + id=self.sensor_definition["computer_spec"]["id"] + ), + sensor_sha512=SENSOR_DEFINITION_HASH, + ) + ) + def test_required_components(self): """Fail acquisition if a required component is not available.""" if not self.sigan.is_available: @@ -987,16 +1059,6 @@ def get_sigmf_builder( sigmf_builder.set_num_channels(len(iter_params)) sigmf_builder.set_task(task_id) - # Add (minimal) ntia-sensor metadata: ID + hash - sigmf_builder.set_sensor( - ntia_sensor.Sensor( - sensor_spec=ntia_core.HardwareSpec( - id=self.sensor_definition["sensor_spec"]["id"], - ), - sensor_sha512=SENSOR_DEFINITION_HASH, - ) - ) - # Mark data as UNCLASSIFIED sigmf_builder.set_classification("UNCLASSIFIED") diff --git a/scos_actions/metadata/sigmf_builder.py b/scos_actions/metadata/sigmf_builder.py index 148b82e9..2341d31d 100644 --- a/scos_actions/metadata/sigmf_builder.py +++ b/scos_actions/metadata/sigmf_builder.py @@ -27,7 +27,7 @@ }, { "name": "ntia-diagnostics", - "version": "v1.0.0", + "version": "v1.1.0", "optional": True, }, { @@ -47,7 +47,7 @@ }, { "name": "ntia-nasctn-sea", - "version": "v0.5.0", + "version": "v0.6.0", "optional": True, }, ], @@ -321,7 +321,7 @@ def set_diagnostics(self, diagnostics: Diagnostics) -> None: """ self.sigmf_md.set_global_field("ntia-diagnostics:diagnostics", diagnostics) - ### ntia-nasctn-sea v0.4.0 ### + ### ntia-nasctn-sea v0.6.0 ### def set_max_of_max_channel_powers( self, max_of_max_channel_powers: List[float] @@ -335,6 +335,26 @@ def set_max_of_max_channel_powers( "ntia-nasctn-sea:max_of_max_channel_powers", max_of_max_channel_powers ) + def set_mean_channel_powers(self, mean_channel_powers: List[float]) -> None: + """ + Set the value of the Global "ntia-nasctn-sea:mean_channel_powers" field. + + :param mean_channel_powers: The mean power per channel, in dBm. + """ + self.sigmf_md.set_global_field( + "ntia-nasctn-sea:mean_channel_powers", mean_channel_powers + ) + + def set_median_channel_powers(self, median_channel_powers: List[float]) -> None: + """ + Set the value of the Global "ntia-nasctn-sea:median_channel_powers" field. + + :param median_channel_powers: The median power per channel, in dBm. + """ + self.sigmf_md.set_global_field( + "ntia-nasctn-sea:median_channel_powers", median_channel_powers + ) + def set_median_of_mean_channel_powers( self, median_of_mean_channel_powers: List[float] ) -> None: diff --git a/scos_actions/metadata/structs/ntia_diagnostics.py b/scos_actions/metadata/structs/ntia_diagnostics.py index 45de8e2a..35dce19c 100644 --- a/scos_actions/metadata/structs/ntia_diagnostics.py +++ b/scos_actions/metadata/structs/ntia_diagnostics.py @@ -119,6 +119,24 @@ class Computer(msgspec.Struct, **SIGMF_OBJECT_KWARGS): ssd_smart_data: Optional[SsdSmartData] = None +class Software(msgspec.Struct, **SIGMF_OBJECT_KWARGS): + """ + Interface for generating `ntia-diagnostics` `Software` objects. + + :param system_platform: The underlying platform, as returned by `platform.platform()` + :param python_version: The Python version, as returned by `sys.version()`. + :param scos_sensor_version: The SCOS Sensor version, as returned by `git describe --tags`. + :param scos_actions_version: Version of `scos_actions` plugin. + :param preselector_api_version: Version of the NTIA `preselector` package. + """ + + system_platform: Optional[str] = None + python_version: Optional[str] = None + scos_sensor_version: Optional[str] = None + scos_actions_version: Optional[str] = None + preselector_api_version: Optional[str] = None + + class Diagnostics(msgspec.Struct, **SIGMF_OBJECT_KWARGS): """ Interface for generating `ntia-diagnostics` `Diagnostics` objects. @@ -135,4 +153,5 @@ class Diagnostics(msgspec.Struct, **SIGMF_OBJECT_KWARGS): preselector: Optional[Preselector] = None spu: Optional[SPU] = None computer: Optional[Computer] = None + software: Optional[Software] = None action_runtime: Optional[float] = None diff --git a/scos_actions/settings.py b/scos_actions/settings.py index e43dd016..8ee0015c 100644 --- a/scos_actions/settings.py +++ b/scos_actions/settings.py @@ -38,13 +38,13 @@ SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None) SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None) MOCK_SIGAN = env("MOCK_SIGAN", default=None) - - + SCOS_SENSOR_GIT_TAG = env("SCOS_SENSOR_GIT_TAG", default="unknown") else: MOCK_SIGAN = settings.MOCK_SIGAN RUNNING_TESTS = settings.RUNNING_TESTS SENSOR_DEFINITION_FILE = Path(settings.SENSOR_DEFINITION_FILE) FQDN = settings.FQDN + SCOS_SENSOR_GIT_TAG = settings.SCOS_SENSOR_GIT_TAG if settings.PRESELECTOR_CONFIG: PRESELECTOR_CONFIG_FILE = settings.PRESELECTOR_CONFIG else: