Update unittest with run success check

qiskit_experiments/framework/composite/composite_analysis.py

@@ -14,6 +14,7 @@
 """
 
 from typing import List, Dict
+
 import numpy as np
 from qiskit.result import marginal_counts
 from qiskit_experiments.framework import BaseAnalysis, ExperimentData

releasenotes/notes/developer_add_success_check-5ddd1d56be29a329.yaml

@@ -0,0 +1,10 @@
+---
+developer:
+  - |
+    :py:func:`assertComplete` and :py:func:`assertFail` functions have been added to
+    :py:class:`test.base.QiskitExperimentsTestCase`. These assertions will check
+    if the whole execution chain is successfuly completed or intentionally failed.
+    These checks are introduced since calling :py:meth:`BaseExperiment.run` will suppress
+    all error during execution and error message are stored in the experiment data
+    error log with status property. It is `highly recommended` to add these assertions
+    right after experiment run method call to detect malfunction of the program.

test/base.py

@@ -15,23 +15,52 @@
 
 import dataclasses
 import json
-from typing import Any, Callable, Optional
 import warnings
+from typing import Any, Callable, Optional
 
 import numpy as np
 from qiskit.test import QiskitTestCase
+from qiskit_experiments.calibration_management import Calibrations
+from qiskit_experiments.database_service.db_experiment_data import ExperimentStatus
 from qiskit_experiments.framework import (
     ExperimentDecoder,
     ExperimentEncoder,
+    ExperimentData,
     BaseExperiment,
     BaseAnalysis,
 )
-from qiskit_experiments.calibration_management import Calibrations
 
 
 class QiskitExperimentsTestCase(QiskitTestCase):
     """Qiskit Experiments specific extra functionality for test cases."""
 
+    def assertComplete(self, experiment_data: ExperimentData):
+        """Assert that an experiment is succeeded.
+
+        Args:
+            experiment_data: Experiment data to evaluate.
+        """
+        experiment_data.block_for_results()
+
+        self.assertEqual(
+            experiment_data.status(),
+            ExperimentStatus.DONE,
+            msg="All threads are executed but status is not DONE. " + experiment_data.errors(),
+        )
+
+    def assertFail(self, experiment_data: ExperimentData):
+        """Assert that an experiment is failed.
+
+        Args:
+            experiment_data: Experiment data to evaluate.
+        """
+        experiment_data.block_for_results()
+        self.assertNotEqual(
+            experiment_data.status(),
+            ExperimentStatus.DONE,
+            msg="This experiment is expected to fail but all threads are completed successfuly.",
+        )
+
     def assertRoundTripSerializable(self, obj: Any, check_func: Optional[Callable] = None):
         """Assert that an object is round trip serializable.
 

test/calibration/experiments/test_drag.py

@@ -61,6 +61,7 @@ def test_end_to_end(self):
         drag = RoughDrag(1, self.x_plus)
 
         expdata = drag.run(backend)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(1)
 
         self.assertTrue(abs(result.value.value - backend.ideal_beta) < self.test_tol)
@@ -73,6 +74,7 @@ def test_end_to_end(self):
         drag = RoughDrag(0, self.x_plus)
         drag.analysis.set_options(p0={"beta": 1.2})
         exp_data = drag.run(backend)
+        self.assertComplete(exp_data)
         result = exp_data.analysis_results(1)
 
         self.assertTrue(abs(result.value.value - backend.ideal_beta) < self.test_tol)
@@ -85,6 +87,7 @@ def test_end_to_end(self):
         drag.set_run_options(shots=200)
         drag.analysis.set_options(p0={"beta": 1.8, "freq0": 0.08, "freq1": 0.16, "freq2": 0.32})
         exp_data = drag.run(backend)
+        self.assertComplete(exp_data)
         result = exp_data.analysis_results(1)
 
         meas_level = exp_data.metadata["job_metadata"][-1]["run_options"]["meas_level"]
@@ -155,7 +158,8 @@ def test_update(self):
         prev_beta = self.cals.get_parameter_value("β", (0,), "x")
         self.assertEqual(prev_beta, 0)
 
-        RoughDragCal(qubit, self.cals, backend=self.backend).run().block_for_results()
+        expdata = RoughDragCal(qubit, self.cals, backend=self.backend).run()
+        self.assertComplete(expdata)
 
         new_beta = self.cals.get_parameter_value("β", (0,), "x")
         self.assertTrue(abs(new_beta - self.backend.ideal_beta) < self.test_tol)

test/calibration/experiments/test_fine_amplitude.py

@@ -51,6 +51,7 @@ def test_end_to_end_under_rotation(self, pi_ratio):
         backend = MockFineAmp(error, np.pi, "xp")
 
         expdata = amp_exp.run(backend)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
 
@@ -72,6 +73,7 @@ def test_end_to_end_over_rotation(self, pi_ratio):
         backend = MockFineAmp(error, np.pi, "xp")
 
         expdata = amp_exp.run(backend)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
 
@@ -215,6 +217,7 @@ def test_run_x_cal(self):
 
         # run the calibration experiment. This should update the amp parameter of x which we test.
         exp_data = amp_cal.run(self.backend)
+        self.assertComplete(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         new_amp = init_amp * np.pi / (np.pi + d_theta)
 
@@ -253,6 +256,7 @@ def test_run_sx_cal(self):
 
         # run the calibration experiment. This should update the amp parameter of x which we test.
         exp_data = amp_cal.run(MockFineAmp(-np.pi * 0.07, np.pi / 2, "sx"))
+        self.assertComplete(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         new_amp = init_amp * (np.pi / 2) / (np.pi / 2 + d_theta)
 

test/calibration/experiments/test_fine_drag.py

@@ -66,13 +66,15 @@ def test_end_to_end(self):
         drag.set_experiment_options(schedule=self.schedule)
         drag.set_transpile_options(basis_gates=["rz", "Drag", "sx"])
         exp_data = drag.run(FineDragTestBackend())
+        self.assertComplete(exp_data)
 
         self.assertEqual(exp_data.analysis_results(0).quality, "good")
 
     def test_end_to_end_no_schedule(self):
         """Test that we can run without a schedule."""
 
         exp_data = FineXDrag(0).run(FineDragTestBackend())
+        self.assertComplete(exp_data)
 
         self.assertEqual(exp_data.analysis_results(0).quality, "good")
 
@@ -127,6 +129,7 @@ def test_update_cals(self):
 
         # run the calibration experiment. This should update the beta parameter of x which we test.
         exp_data = drag_cal.run(self.backend)
+        self.assertComplete(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         sigma = 40
         target_angle = np.pi

test/calibration/experiments/test_fine_frequency.py

@@ -55,7 +55,8 @@ def test_end_to_end(self, freq_shift):
         freq_exp = FineFrequency(0, 160, backend)
         freq_exp.set_transpile_options(inst_map=self.inst_map)
 
-        expdata = freq_exp.run(shots=100).block_for_results()
+        expdata = freq_exp.run(shots=100)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
         dt = backend.configuration().dt
@@ -79,7 +80,8 @@ def test_calibration_version(self):
 
         self.assertAlmostEqual(freq_before, armonk_freq)
 
-        fine_freq.run().block_for_results()
+        expdata = fine_freq.run()
+        self.assertComplete(expdata)
 
         freq_after = self.cals.get_parameter_value(self.cals.__drive_freq_parameter__, 0)
 

test/calibration/experiments/test_rabi.py

@@ -54,6 +54,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(0)
 
         self.assertEqual(result.quality, "good")
@@ -64,6 +65,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(0)
         self.assertEqual(result.quality, "good")
         self.assertTrue(abs(result.value.value[1] - backend.rabi_rate) < test_tol)
@@ -73,6 +75,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 101))
         expdata = rabi.run(backend)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(0)
         self.assertEqual(result.quality, "good")
         self.assertTrue(abs(result.value.value[1] - backend.rabi_rate) < test_tol)
@@ -89,6 +92,7 @@ def test_wrong_processor(self):
         rabi.analysis.set_options(data_processor=DataProcessor(fail_key, []))
         rabi.set_run_options(shots=2)
         data = rabi.run(backend)
+        self.assertFail(data)
         result = data.analysis_results()
 
         self.assertEqual(len(result), 0)
@@ -135,6 +139,7 @@ def test_ef_rabi_end_to_end(self):
         rabi = EFRabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertComplete(expdata)
         result = expdata.analysis_results(1)
 
         self.assertEqual(result.quality, "good")

test/calibration/experiments/test_ramsey_xy.py

@@ -44,6 +44,7 @@ def test_end_to_end(self):
 
         for freq_shift in [2e6, -3e6]:
             test_data = ramsey.run(MockRamseyXY(freq_shift=freq_shift))
+            self.assertComplete(test_data)
             meas_shift = test_data.analysis_results(1).value.value
             self.assertTrue((meas_shift - freq_shift) < abs(test_tol * freq_shift))
 
@@ -62,7 +63,8 @@ def test_update_calibrations(self):
         freq_shift = 4e6
         osc_shift = 2e6
         backend = MockRamseyXY(freq_shift=freq_shift + osc_shift)  # oscillation with 6 MHz
-        FrequencyCal(0, self.cals, backend, osc_freq=osc_shift).run().block_for_results()
+        expdata = FrequencyCal(0, self.cals, backend, osc_freq=osc_shift).run()
+        self.assertComplete(expdata)
 
         # Check that qubit frequency after running the cal is shifted by freq_shift, i.e. 4 MHz.
         f01 = self.cals.get_parameter_value(freq_name, 0)

test/calibration/experiments/test_rough_amplitude.py

@@ -60,7 +60,8 @@ def test_update(self):
         self.assertTrue(np.allclose(self.cals.get_parameter_value("amp", 0, "sx"), 0.25))
 
         rabi_ef = RoughXSXAmplitudeCal(0, self.cals)
-        rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5)).block_for_results()
+        expdata = rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5))
+        self.assertComplete(expdata)
 
         tol = 0.002
         self.assertTrue(abs(self.cals.get_parameter_value("amp", 0, "x") - 0.333) < tol)
@@ -135,7 +136,8 @@ def test_ef_update(self):
         self.assertTrue(np.allclose(self.cals.get_parameter_value("amp", 0, "sx12"), 0.2))
 
         rabi_ef = EFRoughXSXAmplitudeCal(0, self.cals)
-        rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5)).block_for_results()
+        expdata = rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5))
+        self.assertComplete(expdata)
 
         tol = 0.002
         self.assertTrue(abs(self.cals.get_parameter_value("amp", 0, "x12") - 0.333) < tol)

test/calibration/experiments/test_rough_frequency.py

@@ -60,7 +60,8 @@ def test_update_calibrations(self):
 
         frequencies = np.linspace(freq01 - 10.0e6, freq01 + 10.0e6, 21)
 
-        RoughFrequencyCal(0, cals, frequencies).run(backend).block_for_results()
+        expdata = RoughFrequencyCal(0, cals, frequencies).run(backend)
+        self.assertComplete(expdata)
 
         # Check the updated frequency which should be shifted by 5MHz.
         post_freq = cals.get_parameter_value(cals.__drive_freq_parameter__, (0,))

test/calibration/test_update_library.py

@@ -66,6 +66,7 @@ def test_frequency(self):
         spec = QubitSpectroscopy(qubit, frequencies)
         spec.set_run_options(meas_level=MeasLevel.CLASSIFIED)
         exp_data = spec.run(backend)
+        self.assertComplete(exp_data)
         result = exp_data.analysis_results(1)
         value = result.value.value
 

test/database_service/test_db_experiment_data.py

@@ -129,9 +129,9 @@ def test_add_data_job(self):
 
         exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
         exp_data.add_jobs(a_job)
-        exp_data.block_for_results()
+        self.assertComplete(exp_data)
         exp_data.add_jobs(jobs)
-        exp_data.block_for_results()
+        self.assertComplete(exp_data)
         self.assertEqual(expected, [sdata["counts"] for sdata in exp_data.data()])
         self.assertIn(a_job.job_id(), exp_data.job_ids)
 
@@ -156,7 +156,7 @@ def _callback(_exp_data):
         exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
         exp_data.add_jobs(a_job)
         exp_data.add_analysis_callback(_callback)
-        exp_data.block_for_results()
+        self.assertComplete(exp_data)
         self.assertTrue(called_back)
 
     def test_add_data_callback(self):
@@ -189,7 +189,7 @@ def _callback(_exp_data):
             with self.subTest(data=data):
                 exp_data.add_data(data)
                 exp_data.add_analysis_callback(_callback)
-                exp_data.block_for_results()
+                self.assertComplete(exp_data)
 
         self.assertEqual(len(subtests), called_back_count)
 
@@ -211,7 +211,7 @@ def _callback(_exp_data, **kwargs):
         exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
         exp_data.add_jobs(a_job)
         exp_data.add_analysis_callback(_callback, foo=callback_kwargs)
-        exp_data.block_for_results()
+        self.assertComplete(exp_data)
         self.assertTrue(called_back)
 
     def test_add_data_pending_post_processing(self):
@@ -637,8 +637,7 @@ def test_status_done(self):
         exp_data = DbExperimentData(experiment_type="qiskit_test")
         exp_data.add_jobs(job)
         exp_data.add_jobs(job)
-        exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1))
-        exp_data.block_for_results()
+        self.assertComplete(exp_data)
         self.assertEqual(ExperimentStatus.DONE, exp_data.status())
 
     def test_set_tags(self):
@@ -667,7 +666,7 @@ def _job_result():
         # Cleanup
         with self.assertLogs("qiskit_experiments", "WARNING"):
             event.set()
-            exp_data.block_for_results()
+            self.assertComplete(exp_data)
 
     def test_cancel_analysis(self):
         """Test canceling experiment analysis."""
@@ -752,7 +751,7 @@ def _job_result():
         exp_data.add_jobs(job, timeout=0.5)
 
         with self.assertLogs("qiskit_experiments", "WARNING"):
-            exp_data.block_for_results()
+            self.assertFail(exp_data)
             self.assertEqual(exp_data.job_status(), JobStatus.CANCELLED)
             self.assertEqual(exp_data.status(), ExperimentStatus.CANCELLED)
 
@@ -813,7 +812,7 @@ def _sleeper(*args, **kwargs):  # pylint: disable=unused-argument
         exp_data = DbExperimentData(experiment_type="qiskit_test")
         exp_data.add_jobs(job)
         exp_data.add_analysis_callback(_sleeper)
-        exp_data.block_for_results()
+        self.assertComplete(exp_data)
         self.assertEqual(2, sleep_count)
 
     def test_additional_attr(self):

test/quantum_volume/test_qv.py

@@ -105,8 +105,10 @@ def test_qv_sigma_decreasing(self):
         # set number of trials to a low number to make the test faster
         qv_exp.set_experiment_options(trials=2)
         expdata1 = qv_exp.run(backend)
+        self.assertComplete(expdata1)
         result_data1 = expdata1.analysis_results(0)
         expdata2 = qv_exp.run(backend, analysis=None)
+        self.assertComplete(expdata2)
         expdata2.add_data(expdata1.data())
         qv_exp.analysis.run(expdata2)
         result_data2 = expdata2.analysis_results(0)

test/randomized_benchmarking/test_rb.py

@@ -58,6 +58,7 @@ def test_rb_experiment(self, qubits: list):
             seed=exp_attributes["seed"],
         )
         exp_data = rb_exp.run(backend)
+        self.assertComplete(exp_data)
         exp = exp_data.experiment
         exp_circuits = rb_exp.circuits()
         self.validate_metadata(exp_circuits, exp_attributes)
@@ -203,6 +204,7 @@ def test_interleaved_rb_experiment(self, interleaved_element: "Gate", qubits: li
             seed=exp_attributes["seed"],
         )
         experiment_obj = rb_exp.run(backend)
+        self.assertComplete(experiment_obj)
         exp_data = experiment_obj.experiment
         exp_circuits = rb_exp.circuits()
         self.validate_metadata(exp_circuits, exp_attributes)