Update unittest with run success check

releasenotes/notes/developer_add_success_check-5ddd1d56be29a329.yaml

@@ -0,0 +1,10 @@
+---
+developer:
+  - |
+    :py:func:`assertSuccess` and :py:func:`assertFail` functions have been added to
+    :py:class:`test.base.QiskitExperimentsTestCase`. These assertions will check
+    if the whole execution chain is successfuly completed or intentionally failed.
+    These checks are introduced since calling :py:meth:`BaseExperiment.run` will suppress
+    all error during execution and error message are stored in the experiment data
+    error log with status property. It is `highly recommended` to add these assertions
+    right after experiment run method call to detect malfunction of the program.

test/base.py

@@ -18,12 +18,30 @@
 import numpy as np
 
 from qiskit.test import QiskitTestCase
-from qiskit_experiments.framework import ExperimentDecoder, ExperimentEncoder
+from qiskit_experiments.framework import ExperimentDecoder, ExperimentEncoder, ExperimentData
 
 
 class QiskitExperimentsTestCase(QiskitTestCase):
     """Qiskit Experiments specific extra functionality for test cases."""
 
+    def assertSuccess(self, experiment_data: ExperimentData):
+        """Assert that an experiment is succeeded.
+
+        Args:
+            experiment_data: Experiment data to evaluate.
+        """
+        experiment_data.block_for_results()
+        self.assertEqual(experiment_data.status(), "DONE", msg=experiment_data.errors())
+
+    def assertFail(self, experiment_data: ExperimentData):
+        """Assert that an experiment is failed.
+
+        Args:
+            experiment_data: Experiment data to evaluate.
+        """
+        experiment_data.block_for_results()
+        self.assertEqual(experiment_data.status(), "ERROR")
+
     def assertRoundTripSerializable(self, obj: Any, check_func: Optional[Callable] = None):
         """Assert that an object is round trip serializable.
 

test/calibration/experiments/test_drag.py

@@ -61,6 +61,7 @@ def test_end_to_end(self):
         drag = RoughDrag(1, self.x_plus)
 
         expdata = drag.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
 
         self.assertTrue(abs(result.value.value - backend.ideal_beta) < self.test_tol)
@@ -73,6 +74,7 @@ def test_end_to_end(self):
         drag = RoughDrag(0, self.x_plus)
         drag.analysis.set_options(p0={"beta": 1.2})
         exp_data = drag.run(backend)
+        self.assertSuccess(exp_data)
         result = exp_data.analysis_results(1)
 
         self.assertTrue(abs(result.value.value - backend.ideal_beta) < self.test_tol)
@@ -85,6 +87,7 @@ def test_end_to_end(self):
         drag.set_run_options(shots=200)
         drag.analysis.set_options(p0={"beta": 1.8, "freq0": 0.08, "freq1": 0.16, "freq2": 0.32})
         exp_data = drag.run(backend)
+        self.assertSuccess(exp_data)
         result = exp_data.analysis_results(1)
 
         meas_level = exp_data.metadata["job_metadata"][-1]["run_options"]["meas_level"]
@@ -155,7 +158,8 @@ def test_update(self):
         prev_beta = self.cals.get_parameter_value("β", (0,), "x")
         self.assertEqual(prev_beta, 0)
 
-        RoughDragCal(qubit, self.cals, backend=self.backend).run().block_for_results()
+        expdata = RoughDragCal(qubit, self.cals, backend=self.backend).run()
+        self.assertSuccess(expdata)
 
         new_beta = self.cals.get_parameter_value("β", (0,), "x")
         self.assertTrue(abs(new_beta - self.backend.ideal_beta) < self.test_tol)

test/calibration/experiments/test_fine_amplitude.py

@@ -51,6 +51,7 @@ def test_end_to_end_under_rotation(self, pi_ratio):
         backend = MockFineAmp(error, np.pi, "xp")
 
         expdata = amp_exp.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
 
@@ -72,6 +73,7 @@ def test_end_to_end_over_rotation(self, pi_ratio):
         backend = MockFineAmp(error, np.pi, "xp")
 
         expdata = amp_exp.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
 
@@ -215,6 +217,7 @@ def test_run_x_cal(self):
 
         # run the calibration experiment. This should update the amp parameter of x which we test.
         exp_data = amp_cal.run(self.backend)
+        self.assertSuccess(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         new_amp = init_amp * np.pi / (np.pi + d_theta)
 
@@ -253,6 +256,7 @@ def test_run_sx_cal(self):
 
         # run the calibration experiment. This should update the amp parameter of x which we test.
         exp_data = amp_cal.run(MockFineAmp(-np.pi * 0.07, np.pi / 2, "sx"))
+        self.assertSuccess(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         new_amp = init_amp * (np.pi / 2) / (np.pi / 2 + d_theta)
 

test/calibration/experiments/test_fine_drag.py

@@ -66,13 +66,15 @@ def test_end_to_end(self):
         drag.set_experiment_options(schedule=self.schedule)
         drag.set_transpile_options(basis_gates=["rz", "Drag", "sx"])
         exp_data = drag.run(FineDragTestBackend())
+        self.assertSuccess(exp_data)
 
         self.assertEqual(exp_data.analysis_results(0).quality, "good")
 
     def test_end_to_end_no_schedule(self):
         """Test that we can run without a schedule."""
 
         exp_data = FineXDrag(0).run(FineDragTestBackend())
+        self.assertSuccess(exp_data)
 
         self.assertEqual(exp_data.analysis_results(0).quality, "good")
 
@@ -127,6 +129,7 @@ def test_update_cals(self):
 
         # run the calibration experiment. This should update the beta parameter of x which we test.
         exp_data = drag_cal.run(self.backend)
+        self.assertSuccess(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         sigma = 40
         target_angle = np.pi

test/calibration/experiments/test_rabi.py

@@ -54,6 +54,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(0)
 
         self.assertEqual(result.quality, "good")
@@ -64,6 +65,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(0)
         self.assertEqual(result.quality, "good")
         self.assertTrue(abs(result.value.value[1] - backend.rabi_rate) < test_tol)
@@ -73,6 +75,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 101))
         expdata = rabi.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(0)
         self.assertEqual(result.quality, "good")
         self.assertTrue(abs(result.value.value[1] - backend.rabi_rate) < test_tol)
@@ -89,6 +92,7 @@ def test_wrong_processor(self):
         rabi.analysis.set_options(data_processor=DataProcessor(fail_key, []))
         rabi.set_run_options(shots=2)
         data = rabi.run(backend)
+        self.assertFail(data)
         result = data.analysis_results()
 
         self.assertEqual(len(result), 0)
@@ -135,6 +139,7 @@ def test_ef_rabi_end_to_end(self):
         rabi = EFRabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
 
         self.assertEqual(result.quality, "good")

test/calibration/experiments/test_ramsey_xy.py

@@ -44,6 +44,7 @@ def test_end_to_end(self):
 
         for freq_shift in [2e6, -3e6]:
             test_data = ramsey.run(MockRamseyXY(freq_shift=freq_shift))
+            self.assertSuccess(test_data)
             meas_shift = test_data.analysis_results(1).value.value
             self.assertTrue((meas_shift - freq_shift) < abs(test_tol * freq_shift))
 
@@ -62,7 +63,8 @@ def test_update_calibrations(self):
         freq_shift = 4e6
         osc_shift = 2e6
         backend = MockRamseyXY(freq_shift=freq_shift + osc_shift)  # oscillation with 6 MHz
-        FrequencyCal(0, self.cals, backend, osc_freq=osc_shift).run().block_for_results()
+        expdata = FrequencyCal(0, self.cals, backend, osc_freq=osc_shift).run()
+        self.assertSuccess(expdata)
 
         # Check that qubit frequency after running the cal is shifted by freq_shift, i.e. 4 MHz.
         f01 = self.cals.get_parameter_value(freq_name, 0)

test/calibration/experiments/test_rough_amplitude.py

@@ -60,7 +60,8 @@ def test_update(self):
         self.assertTrue(np.allclose(self.cals.get_parameter_value("amp", 0, "sx"), 0.25))
 
         rabi_ef = RoughXSXAmplitudeCal(0, self.cals)
-        rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5)).block_for_results()
+        expdata = rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5))
+        self.assertSuccess(expdata)
 
         tol = 0.002
         self.assertTrue(abs(self.cals.get_parameter_value("amp", 0, "x") - 0.333) < tol)
@@ -135,7 +136,8 @@ def test_ef_update(self):
         self.assertTrue(np.allclose(self.cals.get_parameter_value("amp", 0, "sx12"), 0.2))
 
         rabi_ef = EFRoughXSXAmplitudeCal(0, self.cals)
-        rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5)).block_for_results()
+        expdata = rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5))
+        self.assertSuccess(expdata)
 
         tol = 0.002
         self.assertTrue(abs(self.cals.get_parameter_value("amp", 0, "x12") - 0.333) < tol)

test/calibration/experiments/test_rough_frequency.py

@@ -61,7 +61,8 @@ def test_update_calibrations(self):
 
         frequencies = np.linspace(freq01 - 10.0e6, freq01 + 10.0e6, 21)
 
-        RoughFrequencyCal(0, cals, frequencies).run(backend).block_for_results()
+        expdata = RoughFrequencyCal(0, cals, frequencies).run(backend)
+        self.assertSuccess(expdata)
 
         # Check the updated frequency which should be shifted by 5MHz.
         post_freq = cals.get_parameter_value(cals.__drive_freq_parameter__, (0,))

test/calibration/test_update_library.py

@@ -66,6 +66,7 @@ def test_frequency(self):
         spec = QubitSpectroscopy(qubit, frequencies, unit="MHz")
         spec.set_run_options(meas_level=MeasLevel.CLASSIFIED)
         exp_data = spec.run(backend)
+        self.assertSuccess(exp_data)
         result = exp_data.analysis_results(1)
         value = result.value.value
 

test/quantum_volume/test_qv.py

@@ -105,8 +105,10 @@ def test_qv_sigma_decreasing(self):
         # set number of trials to a low number to make the test faster
         qv_exp.set_experiment_options(trials=2)
         expdata1 = qv_exp.run(backend)
+        self.assertSuccess(expdata1)
         result_data1 = expdata1.analysis_results(0)
         expdata2 = qv_exp.run(backend, analysis=None)
+        self.assertSuccess(expdata2)
         expdata2.add_data(expdata1.data())
         qv_exp.analysis.run(expdata2)
         result_data2 = expdata2.analysis_results(0)

test/randomized_benchmarking/test_rb.py

@@ -58,6 +58,7 @@ def test_rb_experiment(self, qubits: list):
             seed=exp_attributes["seed"],
         )
         exp_data = rb_exp.run(backend)
+        self.assertSuccess(exp_data)
         exp = exp_data.experiment
         exp_circuits = rb_exp.circuits()
         self.validate_metadata(exp_circuits, exp_attributes)
@@ -203,6 +204,7 @@ def test_interleaved_rb_experiment(self, interleaved_element: "Gate", qubits: li
             seed=exp_attributes["seed"],
         )
         experiment_obj = rb_exp.run(backend)
+        self.assertSuccess(experiment_obj)
         exp_data = experiment_obj.experiment
         exp_circuits = rb_exp.circuits()
         self.validate_metadata(exp_circuits, exp_attributes)

test/test_composite.py

@@ -64,7 +64,8 @@ def test_parallel_options(self):
             self.assertEqual(par_exp.transpile_options, Options(optimization_level=0))
             self.assertEqual(par_exp.analysis.options, Options())
 
-            par_exp.run(FakeBackend())
+            expdata = par_exp.run(FakeBackend())
+            self.assertSuccess(expdata)
 
 
 class TestCompositeExperimentData(QiskitExperimentsTestCase):
@@ -174,15 +175,18 @@ def test_analysis_replace_results_true(self):
         exp1 = FakeExperiment([0, 2])
         exp2 = FakeExperiment([1, 3])
         par_exp = ParallelExperiment([exp1, exp2])
-        data1 = par_exp.run(FakeBackend()).block_for_results()
+        data1 = par_exp.run(FakeBackend())
+        self.assertSuccess(data1)
 
         # Additional data not part of composite experiment
         exp3 = FakeExperiment([0, 1])
         extra_data = exp3.run(FakeBackend())
+        self.assertSuccess(extra_data)
         data1.add_child_data(extra_data)
 
         # Replace results
         data2 = par_exp.analysis.run(data1, replace_results=True)
+        self.assertSuccess(data2)
         self.assertEqual(data1, data2)
         self.assertEqual(len(data1.child_data()), len(data2.child_data()))
         for sub1, sub2 in zip(data1.child_data(), data2.child_data()):
@@ -195,15 +199,18 @@ def test_analysis_replace_results_false(self):
         exp1 = FakeExperiment([0, 2])
         exp2 = FakeExperiment([1, 3])
         par_exp = BatchExperiment([exp1, exp2])
-        data1 = par_exp.run(FakeBackend()).block_for_results()
+        data1 = par_exp.run(FakeBackend())
+        self.assertSuccess(data1)
 
         # Additional data not part of composite experiment
         exp3 = FakeExperiment([0, 1])
         extra_data = exp3.run(FakeBackend())
+        self.assertSuccess(extra_data)
         data1.add_child_data(extra_data)
 
         # Replace results
         data2 = par_exp.analysis.run(data1, replace_results=False)
+        self.assertSuccess(data2)
         self.assertNotEqual(data1.experiment_id, data2.experiment_id)
         self.assertEqual(len(data1.child_data()), len(data2.child_data()))
         for sub1, sub2 in zip(data1.child_data(), data2.child_data()):
@@ -216,7 +223,8 @@ def test_composite_tags(self):
         exp1 = FakeExperiment([0, 2])
         exp2 = FakeExperiment([1, 3])
         par_exp = BatchExperiment([exp1, exp2])
-        expdata = par_exp.run(FakeBackend()).block_for_results()
+        expdata = par_exp.run(FakeBackend())
+        self.assertSuccess(expdata)
         data1 = expdata.child_data(0)
         data2 = expdata.child_data(1)
 
@@ -372,7 +380,8 @@ def circuits(self):
         par_exp = ParallelExperiment(
             [exp1, BatchExperiment([ParallelExperiment([exp2, exp3]), exp4])]
         )
-        expdata = par_exp.run(Backend()).block_for_results()
+        expdata = par_exp.run(Backend())
+        self.assertSuccess(expdata)
 
         for circ_data, circ_counts in zip(expdata.data(), counts):
             self.assertDictEqual(circ_data["counts"], circ_counts)

test/test_cross_resonance_hamiltonian.py

@@ -318,6 +318,7 @@ def test_integration(self, ix, iy, iz, zx, zy, zz):
             qubits=(0, 1), flat_top_widths=durations, sigma=sigma, risefall=2
         )
         exp_data = expr.run(backend, shots=2000)
+        self.assertSuccess(exp_data)
 
         self.assertEqual(exp_data.analysis_results(0).quality, "good")
         self.assertAlmostEqual(exp_data.analysis_results("omega_ix").value.value, ix, delta=2e4)

test/test_framework.py

@@ -43,6 +43,7 @@ def circuits(self):
 
         exp = Experiment([0])
         expdata = exp.run(backend)
+        self.assertSuccess(expdata)
         job_ids = expdata.job_ids
 
         # Comptue expected number of jobs

test/test_half_angle.py

@@ -51,6 +51,7 @@ def test_end_to_end(self):
         for error in [-0.05, -0.02, 0.02, 0.05]:
             hac = HalfAngle(0)
             exp_data = hac.run(HalfAngleTestBackend(error))
+            self.assertSuccess(exp_data)
             d_theta = exp_data.analysis_results(1).value.value
 
             self.assertTrue(abs(d_theta - error) < tol)

test/test_qubit_spectroscopy.py

@@ -64,6 +64,7 @@ def test_spectroscopy_end2end_classified(self):
         spec = QubitSpectroscopy(qubit, frequencies, unit="Hz")
         spec.set_run_options(meas_level=MeasLevel.CLASSIFIED)
         expdata = spec.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         value = result.value.value
 
@@ -76,6 +77,7 @@ def test_spectroscopy_end2end_classified(self):
         spec = QubitSpectroscopy(qubit, frequencies, unit="Hz")
         spec.set_run_options(meas_level=MeasLevel.CLASSIFIED)
         expdata = spec.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         value = result.value.value
 
@@ -92,6 +94,7 @@ def test_spectroscopy_end2end_kerneled(self):
 
         spec = QubitSpectroscopy(qubit, frequencies, unit="MHz")
         expdata = spec.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         value = result.value.value
 
@@ -103,6 +106,7 @@ def test_spectroscopy_end2end_kerneled(self):
 
         spec = QubitSpectroscopy(qubit, frequencies, unit="MHz")
         expdata = spec.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         value = result.value.value
 
@@ -111,6 +115,7 @@ def test_spectroscopy_end2end_kerneled(self):
 
         spec.set_run_options(meas_return="avg")
         expdata = spec.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         value = result.value.value
 
@@ -131,6 +136,7 @@ def test_spectroscopy12_end2end_classified(self):
         spec.backend = backend
         spec.set_run_options(meas_level=MeasLevel.CLASSIFIED)
         expdata = spec.run(backend)
+        self.assertSuccess(expdata)
         result = expdata.analysis_results(1)
         value = result.value.value