Add simple one-time retry on failed calibration

scos_actions/actions/calibrate_y_factor.py

@@ -255,10 +255,19 @@ def __call__(self, sensor: Sensor, schedule_entry: dict, task_id: int):
 
         # Run calibration routine
         for i, p in enumerate(self.iteration_params):
+            cal_result = self.calibrate(p)
+            # Retry once if channel calibration failed
+            if cal_result == "FAILED":
+                logger.warning(f"Retrying calibration at {p[FREQUENCY]/1e6} MHz")
+                cal_result = self.calibrate(p)
+                if cal_result == "FAILED":
+                    logger.warning(
+                        f"Retry failed. Calibration data not updated for f={p[FREQUENCY]}"
+                    )
             if i == 0:
-                detail += self.calibrate(p)
+                detail += cal_result
             else:
-                detail += os.linesep + self.calibrate(p)
+                detail += os.linesep + cal_result
         return detail
 
     def calibrate(self, params: dict):
@@ -346,14 +355,25 @@ def calibrate(self, params: dict):
             pwr_on_watts, pwr_off_watts, enr_linear, enbw_hz, temp_k
         )
 
-        if not np.isnan(gain) and not np.isnan(noise_figure):
+        # TODO: For testing, manually trigger NaN behavior on certain channels
+        if params[FREQUENCY] == 3535e6:
+            gain = np.nan
+        if params[FREQUENCY] == 3545e6:
+            noise_figure = np.nan
+        if params[FREQUENCY] == 3555e6:
+            noise_figure = np.inf
+
+        if np.isfinite(gain) and np.isfinite(noise_figure):
             # Update sensor calibration with results
             self.sensor.sensor_calibration.update(
                 sigan_params, utils.get_datetime_str_now(), gain, noise_figure, temp_c
             )
         else:
-            logger.warning(f"Calibration result is NaN at {params[FREQUENCY]}:")
+            # At least one of {noise figure, gain} is NaN or infinite. This triggers
+            # a single retry for this set of params. See __call__ above.
+            logger.warning(f"Calibration result is NaN at {params[FREQUENCY]/1e6} MHz:")
             logger.warning(f"\tNF: {noise_figure}, Gain: {gain}")
+            return "FAILED"
         # Debugging
         noise_floor_dBm = convert_watts_to_dBm(Boltzmann * temp_k * enbw_hz)
         logger.debug(f"Noise floor: {noise_floor_dBm:.2f} dBm")