diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27a4ee27..9618961d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,19 +23,19 @@ repos: - id: pyupgrade args: ["--py38-plus"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) types: [file, python] args: ["--profile", "black", "--filter-files", "--gitignore"] - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 23.12.1 hooks: - id: black types: [file, python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.37.0 + rev: v0.38.0 hooks: - id: markdownlint types: [file, markdown] diff --git a/pyproject.toml b/pyproject.toml index fda627c5..a34afe30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ "numpy>=1.22.0", "psutil>=5.9.4", "python-dateutil>=2.0", - "ray>=2.4.0", + "ray>=2.6.3,<2.8.0", "ruamel.yaml>=0.15", "scipy>=1.8.0", "sigmf @ git+https://github.com/NTIA/SigMF@multi-recording-archive", diff --git a/scos_actions/__init__.py b/scos_actions/__init__.py index 54ccb5d2..87c6f8ed 100644 --- a/scos_actions/__init__.py +++ b/scos_actions/__init__.py @@ -1 +1,2 @@ __version__ = "7.1.0" + diff --git a/scos_actions/actions/acquire_sea_data_product.py b/scos_actions/actions/acquire_sea_data_product.py index ade1c917..17adb08c 100644 --- a/scos_actions/actions/acquire_sea_data_product.py +++ b/scos_actions/actions/acquire_sea_data_product.py @@ -110,7 +110,8 @@ # Constants DATA_TYPE = np.half PFP_FRAME_RESOLUTION_S = (1e-3 * (1 + 1 / (14)) / 15) / 4 -FFT_SIZE = 875 +FFT_SIZE = 175 # 80 kHz resolution @ 14 MHz sampling rate +FFT_PERCENTILES = np.array([25, 75, 90, 95, 99, 99.9, 99.99]) FFT_WINDOW_TYPE = "flattop" FFT_WINDOW = get_fft_window(FFT_WINDOW_TYPE, FFT_SIZE) FFT_WINDOW_ECF = get_fft_window_correction(FFT_WINDOW, "energy") @@ -119,8 +120,8 @@ NUM_ACTORS = 3 # Number of ray actors to initialize # Create power detectors -TD_DETECTOR = create_statistical_detector("TdMeanMaxDetector", ["mean", "max"]) -FFT_DETECTOR = create_statistical_detector("FftMeanMaxDetector", ["mean", "max"]) +TD_DETECTOR = create_statistical_detector("TdMeanMaxDetector", ["max", "mean"]) +FFT_M3_DETECTOR = create_statistical_detector("FftM3Detector", ["max", "mean", "median"]) PFP_M3_DETECTOR = create_statistical_detector("PfpM3Detector", ["min", "max", "mean"]) @@ -142,20 +143,22 @@ def __init__( fft_size: int = FFT_SIZE, fft_window: np.ndarray = FFT_WINDOW, window_ecf: float = FFT_WINDOW_ECF, - detector: EnumMeta = FFT_DETECTOR, + detector: EnumMeta = FFT_M3_DETECTOR, + percentiles: np.ndarray = FFT_PERCENTILES, impedance_ohms: float = IMPEDANCE_OHMS, ): self.detector = detector + self.percentiles = percentiles self.fft_size = fft_size self.fft_window = fft_window self.num_ffts = num_ffts - # Get truncation points: truncate FFT result to middle 625 samples (middle 10 MHz from 14 MHz) - self.bin_start = int(fft_size / 7) # bin_start = 125 with FFT_SIZE 875 - self.bin_end = fft_size - self.bin_start # bin_end = 750 with FFT_SIZE 875 + # Get truncation points: truncate FFT result to middle 125 samples (middle 10 MHz from 14 MHz) + self.bin_start = int(fft_size / 7) # bin_start = 25 with FFT_SIZE 175 + self.bin_end = fft_size - self.bin_start # bin_end = 150 with FFT_SIZE 175 # Compute the amplitude shift for PSD scaling. The FFT result # is in pseudo-power log units and must be scaled to a PSD. self.fft_scale_factor = ( - -10.0 * np.log10(impedance_ohms) # Pseudo-power to power + - 10.0 * np.log10(impedance_ohms) # Pseudo-power to power + 27.0 # Watts to dBm (+30) and baseband to RF (-3) - 10.0 * np.log10(sample_rate_Hz * fft_size) # PSD scaling + 20.0 * np.log10(window_ecf) # Window energy correction @@ -170,20 +173,23 @@ def run(self, iq: ray.ObjectRef) -> np.ndarray: :return: A 2D NumPy array of statistical detector results computed from PSD amplitudes, ordered (max, mean). """ - fft_result = get_fft( + fft_amplitudes = get_fft( iq, self.fft_size, "backward", self.fft_window, self.num_ffts, False, 1 ) - fft_result = calculate_pseudo_power(fft_result) - fft_result = apply_statistical_detector( - fft_result, self.detector - ) # (max, mean) + # Power in Watts + fft_amplitudes = calculate_pseudo_power(fft_amplitudes) + fft_result = apply_statistical_detector(fft_amplitudes, self.detector) # (max, mean, median) + percentile_result = np.percentile(fft_amplitudes, self.percentiles, axis=0) + fft_result = np.vstack((fft_result, percentile_result)) fft_result = np.fft.fftshift(fft_result, axes=(1,)) # Shift frequencies fft_result = fft_result[ :, self.bin_start : self.bin_end ] # Truncation to middle bins fft_result = 10.0 * np.log10(fft_result) + self.fft_scale_factor - # Returned order is (max, mean) + # Returned order is (max, mean, median, 25%, 75%, 90%, 95%, 99%, 99.9%, 99.99%) + # Total of 10 arrays, each of length 125 (output shape (10, 125)) + # Percentile computation linearly interpolates. See numpy documentation. return fft_result @@ -988,18 +994,15 @@ def create_global_data_product_metadata(self) -> None: self.sigmf_builder.set_processing_info([iir_obj, dft_obj]) psd_length = int(FFT_SIZE * (5 / 7)) - psd_bin_center_offset = p[SAMPLE_RATE] / FFT_SIZE / 2 - psd_x_axis__Hz = np.arange(psd_length) * ( - (p[SAMPLE_RATE] / FFT_SIZE) - - (p[SAMPLE_RATE] * (5 / 7) / 2) - + psd_bin_center_offset - ) psd_bin_start = int(FFT_SIZE / 7) # bin_start = 125 with FFT_SIZE 875 psd_bin_end = FFT_SIZE - psd_bin_start # bin_end = 750 with FFT_SIZE 875 - psd_x_axis__Hz = get_fft_frequencies(FFT_SIZE, 14e6, 0.0) # Baseband + psd_x_axis__Hz = get_fft_frequencies(FFT_SIZE, p[SAMPLE_RATE], 0.0) # Baseband psd_graph = ntia_algorithm.Graph( name="Power Spectral Density", - series=[d.value for d in FFT_DETECTOR], # ["max", "mean"] + series=[d.value for d in FFT_M3_DETECTOR] + + [ + f"{int(p)}th_percentile" if p.is_integer() else f"{p}th_percentile" for p in FFT_PERCENTILES + ], # ["max", "mean", "median", "25th_percentile", "75th_percentile", ... "99.99th_percentile"] length=int(FFT_SIZE * (5 / 7)), x_units="Hz", x_start=[psd_x_axis__Hz[psd_bin_start]], @@ -1009,9 +1012,10 @@ def create_global_data_product_metadata(self) -> None: processing=[dft_obj.id], reference=DATA_REFERENCE_POINT, description=( - "Max- and mean-detected power spectral density, with the " - + f"first and last {int(FFT_SIZE / 7)} samples discarded. " - + "FFTs computed on IIR-filtered data." + "Results of statistical detectors (max, mean, median, 25th_percentile, 75th_percentile, " + + "90th_percentile, 95th_percentile, 99th_percentile, 99.9th_percentile, 99.99th_percentile) " + + f"applied to power spectral density samples, with the first and last {int(FFT_SIZE / 7)} " + + "samples discarded. FFTs computed on IIR-filtered data." ), ) @@ -1089,7 +1093,7 @@ def create_global_data_product_metadata(self) -> None: [psd_graph, pvt_graph, pfp_graph, apd_graph] ) self.total_channel_data_length = ( - psd_length * len(FFT_DETECTOR) + psd_length * (len(FFT_M3_DETECTOR) + len(FFT_PERCENTILES)) + pvt_length * len(TD_DETECTOR) + pfp_length * len(PFP_M3_DETECTOR) * 2 + apd_graph.length