Merge pull request #88 from NTIA/sea-metadata-v0.6

Add new diagnostics and channel power summaries to SEA action metadata

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.4.0
+    rev: v3.10.1
     hooks:
       - id: pyupgrade
         args: ["--py38-plus"]
@@ -30,12 +30,12 @@ repos:
         types: [file, python]
         args: ["--profile", "black", "--filter-files", "--gitignore"]
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
       - id: black
         types: [file, python]
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.34.0
+    rev: v0.35.0
     hooks:
       - id: markdownlint
         types: [file, markdown]

scos_actions/__init__.py

@@ -1 +1 @@
-__version__ = "6.3.2"
+__version__ = "6.3.3"

scos_actions/actions/acquire_sea_data_product.py

@@ -20,18 +20,22 @@
 
 Currently in development.
 """
-import gc
 import logging
 import lzma
+import platform
+import sys
 from enum import EnumMeta
 from time import perf_counter
 from typing import Tuple
 
 import numpy as np
 import psutil
 import ray
+from environs import Env
+from its_preselector import __version__ as PRESELECTOR_API_VERSION
 from scipy.signal import sos2tf, sosfilt
 
+from scos_actions import __version__ as SCOS_ACTIONS_VERSION
 from scos_actions import utils
 from scos_actions.actions.interfaces.action import Action
 from scos_actions.capabilities import SENSOR_DEFINITION_HASH, SENSOR_LOCATION
@@ -53,6 +57,7 @@
     ntia_sensor,
 )
 from scos_actions.metadata.structs.capture import CaptureSegment
+from scos_actions.settings import SCOS_SENSOR_GIT_TAG
 from scos_actions.signal_processing.apd import get_apd
 from scos_actions.signal_processing.fft import (
     get_fft,
@@ -74,6 +79,7 @@
 from scos_actions.status import start_time
 from scos_actions.utils import convert_datetime_to_millisecond_iso_format, get_days_up
 
+env = Env()
 logger = logging.getLogger(__name__)
 
 if not ray.is_initialized():
@@ -271,24 +277,34 @@ def run(self, iq: ray.ObjectRef) -> Tuple[np.ndarray, np.ndarray]:
         :return: Two NumPy arrays: the first has shape (2, 400) and
             the second is 1D with length 2. The first array contains
             the (max, mean) detector results and the second array contains
-            the (max-of-max, median-of-mean) summary statistics.
+            the (max-of-max, median-of-mean, mean, median) single-valued
+            summary statistics.
         """
         # Reshape IQ data into blocks and calculate power
         n_blocks = len(iq) // self.block_size
         iq_pwr = calculate_power_watts(
             iq.reshape((n_blocks, self.block_size)), self.impedance_ohms
         )
+        # Get true median power
+        pvt_median = np.median(iq_pwr.flatten())
         # Apply max/mean detectors
         pvt_result = apply_statistical_detector(iq_pwr, self.detector, axis=1)
-        # Get single value median/max statistics
-        pvt_summary = np.array([pvt_result[0].max(), np.median(pvt_result[1])])
+        # Get single value statistics: (max-of-max, median-of-mean, mean, median)
+        pvt_summary = np.array(
+            [
+                pvt_result[0].max(),
+                np.median(pvt_result[1]),
+                pvt_result[1].mean(),
+                pvt_median,
+            ]
+        )
         # Convert to dBm and account for RF/baseband power difference
         # Note: convert_watts_to_dBm is not used to avoid NumExpr usage
         # for the relatively small arrays
         pvt_result, pvt_summary = (
             10.0 * np.log10(x) + 27.0 for x in [pvt_result, pvt_summary]
         )
-        # Return order ((max array, mean array), (max-of-max, median-of-mean))
+        # Return order ((max array, mean array), (max-of-max, median-of-mean, mean, median))
         return pvt_result, pvt_summary
 
 
@@ -512,6 +528,7 @@ def __call__(self, schedule_entry, task_id):
             schedule_entry,
             self.iteration_params,
         )
+        self.create_global_sensor_metadata()
         self.create_global_data_product_metadata()
 
         # Initialize remote supervisor actors for IQ processing
@@ -550,7 +567,13 @@ def __call__(self, schedule_entry, task_id):
         )
 
         # Collect processed data product results
-        all_data, max_max_ch_pwrs, med_mean_ch_pwrs = [], [], []
+        all_data, max_max_ch_pwrs, med_mean_ch_pwrs, mean_ch_pwrs, median_ch_pwrs = (
+            [],
+            [],
+            [],
+            [],
+            [],
+        )
         result_tic = perf_counter()
         for i, channel_data_process in enumerate(dp_procs):
             # Retrieve object references for channel data
@@ -564,6 +587,8 @@ def __call__(self, schedule_entry, task_id):
                     data, summaries = data  # Split the tuple
                     max_max_ch_pwrs.append(DATA_TYPE(summaries[0]))
                     med_mean_ch_pwrs.append(DATA_TYPE(summaries[1]))
+                    mean_ch_pwrs.append(DATA_TYPE(summaries[2]))
+                    median_ch_pwrs.append(DATA_TYPE(summaries[3]))
                     del summaries
                 if i == 3:  # Separate condition is intentional
                     # APD result: append instead of extend,
@@ -585,6 +610,8 @@ def __call__(self, schedule_entry, task_id):
         all_data = self.compress_bytes_data(np.array(all_data).tobytes())
         self.sigmf_builder.set_max_of_max_channel_powers(max_max_ch_pwrs)
         self.sigmf_builder.set_median_of_mean_channel_powers(med_mean_ch_pwrs)
+        self.sigmf_builder.set_mean_channel_powers(mean_ch_pwrs)
+        self.sigmf_builder.set_median_channel_powers(median_ch_pwrs)
         # Get diagnostics last to record action runtime
         self.capture_diagnostics(
             action_start_tic, cpu_speed
@@ -645,6 +672,9 @@ def capture_diagnostics(self, action_start_tic: float, cpu_speeds: list) -> None
             CPU temperature, CPU overheating status, CPU uptime, SCOS
             start time, and SCOS uptime.
 
+        Software versions: the OS platform, Python version, scos_actions
+            version, and preselector API version.
+
         The total action runtime is also recorded.
 
         Preselector X410 Setup requires:
@@ -670,6 +700,8 @@ def capture_diagnostics(self, action_start_tic: float, cpu_speeds: list) -> None
 
         :param action_start_tic: Action start timestamp, as would
              be returned by ``time.perf_counter()``
+        :param cpu_speeds: List of CPU speed values, recorded at
+            consecutive points as the action has been running.
         """
         tic = perf_counter()
         # Read SPU sensors
@@ -758,19 +790,59 @@ def capture_diagnostics(self, action_start_tic: float, cpu_speeds: list) -> None
         except:
             logger.warning("Failed to get SSD SMART data")
 
+        # Get software versions
+        software_diag = {
+            "system_platform": platform.platform(),
+            "python_version": sys.version.split()[0],
+            "scos_sensor_version": SCOS_SENSOR_GIT_TAG,
+            "scos_actions_version": SCOS_ACTIONS_VERSION,
+            "preselector_api_version": PRESELECTOR_API_VERSION,
+        }
+
         toc = perf_counter()
         logger.debug(f"Got all diagnostics in {toc-tic} s")
         diagnostics = {
             "datetime": utils.get_datetime_str_now(),
             "preselector": ntia_diagnostics.Preselector(**ps_diag),
             "spu": ntia_diagnostics.SPU(**spu_diag),
             "computer": ntia_diagnostics.Computer(**cpu_diag),
+            "software": ntia_diagnostics.Software(**software_diag),
             "action_runtime": round(perf_counter() - action_start_tic, 2),
         }
 
         # Add diagnostics to SigMF global object
         self.sigmf_builder.set_diagnostics(ntia_diagnostics.Diagnostics(**diagnostics))
 
+    def create_global_sensor_metadata(self):
+        # Add (minimal) ntia-sensor metadata to the sigmf_builder:
+        #   sensor ID, serial numbers for preselector, sigan, and computer
+        #   overall sensor_spec version, e.g. "Prototype Rev. 3"
+        #   sensor definition hash, to link to full sensor definition
+        self.sigmf_builder.set_sensor(
+            ntia_sensor.Sensor(
+                sensor_spec=ntia_core.HardwareSpec(
+                    id=self.sensor_definition["sensor_spec"]["id"],
+                    version=self.sensor_definition["sensor_spec"]["version"],
+                ),
+                preselector=ntia_sensor.Preselector(
+                    preselector_spec=ntia_core.HardwareSpec(
+                        id=self.sensor_definition["preselector"]["preselector_spec"][
+                            "id"
+                        ]
+                    )
+                ),
+                signal_analyzer=ntia_sensor.SignalAnalyzer(
+                    sigan_spec=ntia_core.HardwareSpec(
+                        id=self.sensor_definition["signal_analyzer"]["sigan_spec"]["id"]
+                    )
+                ),
+                computer_spec=ntia_sensor.HardwareSpec(
+                    id=self.sensor_definition["computer_spec"]["id"]
+                ),
+                sensor_sha512=SENSOR_DEFINITION_HASH,
+            )
+        )
+
     def test_required_components(self):
         """Fail acquisition if a required component is not available."""
         if not self.sigan.is_available:
@@ -987,16 +1059,6 @@ def get_sigmf_builder(
         sigmf_builder.set_num_channels(len(iter_params))
         sigmf_builder.set_task(task_id)
 
-        # Add (minimal) ntia-sensor metadata: ID + hash
-        sigmf_builder.set_sensor(
-            ntia_sensor.Sensor(
-                sensor_spec=ntia_core.HardwareSpec(
-                    id=self.sensor_definition["sensor_spec"]["id"],
-                ),
-                sensor_sha512=SENSOR_DEFINITION_HASH,
-            )
-        )
-
         # Mark data as UNCLASSIFIED
         sigmf_builder.set_classification("UNCLASSIFIED")
 

scos_actions/metadata/sigmf_builder.py

@@ -27,7 +27,7 @@
         },
         {
             "name": "ntia-diagnostics",
-            "version": "v1.0.0",
+            "version": "v1.1.0",
             "optional": True,
         },
         {
@@ -47,7 +47,7 @@
         },
         {
             "name": "ntia-nasctn-sea",
-            "version": "v0.5.0",
+            "version": "v0.6.0",
             "optional": True,
         },
     ],
@@ -321,7 +321,7 @@ def set_diagnostics(self, diagnostics: Diagnostics) -> None:
         """
         self.sigmf_md.set_global_field("ntia-diagnostics:diagnostics", diagnostics)
 
-    ### ntia-nasctn-sea v0.4.0 ###
+    ### ntia-nasctn-sea v0.6.0 ###
 
     def set_max_of_max_channel_powers(
         self, max_of_max_channel_powers: List[float]
@@ -335,6 +335,26 @@ def set_max_of_max_channel_powers(
             "ntia-nasctn-sea:max_of_max_channel_powers", max_of_max_channel_powers
         )
 
+    def set_mean_channel_powers(self, mean_channel_powers: List[float]) -> None:
+        """
+        Set the value of the Global "ntia-nasctn-sea:mean_channel_powers" field.
+
+        :param mean_channel_powers: The mean power per channel, in dBm.
+        """
+        self.sigmf_md.set_global_field(
+            "ntia-nasctn-sea:mean_channel_powers", mean_channel_powers
+        )
+
+    def set_median_channel_powers(self, median_channel_powers: List[float]) -> None:
+        """
+        Set the value of the Global "ntia-nasctn-sea:median_channel_powers" field.
+
+        :param median_channel_powers: The median power per channel, in dBm.
+        """
+        self.sigmf_md.set_global_field(
+            "ntia-nasctn-sea:median_channel_powers", median_channel_powers
+        )
+
     def set_median_of_mean_channel_powers(
         self, median_of_mean_channel_powers: List[float]
     ) -> None:

scos_actions/metadata/structs/ntia_diagnostics.py

@@ -119,6 +119,24 @@ class Computer(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
     ssd_smart_data: Optional[SsdSmartData] = None
 
 
+class Software(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
+    """
+    Interface for generating `ntia-diagnostics` `Software` objects.
+
+    :param system_platform: The underlying platform, as returned by `platform.platform()`
+    :param python_version: The Python version, as returned by `sys.version()`.
+    :param scos_sensor_version: The SCOS Sensor version, as returned by `git describe --tags`.
+    :param scos_actions_version: Version of `scos_actions` plugin.
+    :param preselector_api_version: Version of the NTIA `preselector` package.
+    """
+
+    system_platform: Optional[str] = None
+    python_version: Optional[str] = None
+    scos_sensor_version: Optional[str] = None
+    scos_actions_version: Optional[str] = None
+    preselector_api_version: Optional[str] = None
+
+
 class Diagnostics(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
     """
     Interface for generating `ntia-diagnostics` `Diagnostics` objects.
@@ -135,4 +153,5 @@ class Diagnostics(msgspec.Struct, **SIGMF_OBJECT_KWARGS):
     preselector: Optional[Preselector] = None
     spu: Optional[SPU] = None
     computer: Optional[Computer] = None
+    software: Optional[Software] = None
     action_runtime: Optional[float] = None

scos_actions/settings.py

@@ -38,13 +38,13 @@
     SIGAN_POWER_CYCLE_STATES = env("SIGAN_POWER_CYCLE_STATES", default=None)
     SIGAN_POWER_SWITCH = env("SIGAN_POWER_SWITCH", default=None)
     MOCK_SIGAN = env("MOCK_SIGAN", default=None)
-
-
+    SCOS_SENSOR_GIT_TAG = env("SCOS_SENSOR_GIT_TAG", default="unknown")
 else:
     MOCK_SIGAN = settings.MOCK_SIGAN
     RUNNING_TESTS = settings.RUNNING_TESTS
     SENSOR_DEFINITION_FILE = Path(settings.SENSOR_DEFINITION_FILE)
     FQDN = settings.FQDN
+    SCOS_SENSOR_GIT_TAG = settings.SCOS_SENSOR_GIT_TAG
     if settings.PRESELECTOR_CONFIG:
         PRESELECTOR_CONFIG_FILE = settings.PRESELECTOR_CONFIG
     else: