Update unittest with run success check

qiskit_experiments/framework/composite/composite_analysis.py

@@ -14,6 +14,7 @@
 """
 
 from typing import List, Dict
+
 import numpy as np
 from qiskit.result import marginal_counts
 from qiskit_experiments.framework import BaseAnalysis, ExperimentData

releasenotes/notes/developer_add_success_check-5ddd1d56be29a329.yaml

@@ -0,0 +1,10 @@
+---
+developer:
+  - |
+    :py:func:`assertExperimentDone` function has been added to
+    :py:class:`test.base.QiskitExperimentsTestCase`. This assertion will check
+    if all threads in the experiment data are successfuly completed.
+    This function calls `block_for_results` method and then checks if the experiment
+    status returns DONE after execution. It is `highly recommended` to use this test
+    right after each experiment execution to detect program malfunction,
+    which is paticulaly relavant to python multi-threading in multi-platform.

test/base.py

@@ -15,23 +15,40 @@
 
 import dataclasses
 import json
-from typing import Any, Callable, Optional
 import warnings
+from typing import Any, Callable, Optional
 
 import numpy as np
 from qiskit.test import QiskitTestCase
+from qiskit_experiments.calibration_management import Calibrations
+from qiskit_experiments.database_service.db_experiment_data import ExperimentStatus
 from qiskit_experiments.framework import (
     ExperimentDecoder,
     ExperimentEncoder,
+    ExperimentData,
     BaseExperiment,
     BaseAnalysis,
 )
-from qiskit_experiments.calibration_management import Calibrations
 
 
 class QiskitExperimentsTestCase(QiskitTestCase):
     """Qiskit Experiments specific extra functionality for test cases."""
 
+    def assertExperimentDone(self, experiment_data: ExperimentData):
+        """Blocking execution of next line until all threads are completed then
+        checks if status returns Done.
+
+        Args:
+            experiment_data: Experiment data to evaluate.
+        """
+        experiment_data.block_for_results()
+
+        self.assertEqual(
+            experiment_data.status(),
+            ExperimentStatus.DONE,
+            msg="All threads are executed but status is not DONE. " + experiment_data.errors(),
+        )
+
     def assertRoundTripSerializable(self, obj: Any, check_func: Optional[Callable] = None):
         """Assert that an object is round trip serializable.
 

test/calibration/experiments/test_drag.py

@@ -61,6 +61,7 @@ def test_end_to_end(self):
         drag = RoughDrag(1, self.x_plus)
 
         expdata = drag.run(backend)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(1)
 
         self.assertTrue(abs(result.value.value - backend.ideal_beta) < self.test_tol)
@@ -73,6 +74,7 @@ def test_end_to_end(self):
         drag = RoughDrag(0, self.x_plus)
         drag.analysis.set_options(p0={"beta": 1.2})
         exp_data = drag.run(backend)
+        self.assertExperimentDone(exp_data)
         result = exp_data.analysis_results(1)
 
         self.assertTrue(abs(result.value.value - backend.ideal_beta) < self.test_tol)
@@ -85,6 +87,7 @@ def test_end_to_end(self):
         drag.set_run_options(shots=200)
         drag.analysis.set_options(p0={"beta": 1.8, "freq0": 0.08, "freq1": 0.16, "freq2": 0.32})
         exp_data = drag.run(backend)
+        self.assertExperimentDone(exp_data)
         result = exp_data.analysis_results(1)
 
         meas_level = exp_data.metadata["job_metadata"][-1]["run_options"]["meas_level"]
@@ -155,7 +158,8 @@ def test_update(self):
         prev_beta = self.cals.get_parameter_value("β", (0,), "x")
         self.assertEqual(prev_beta, 0)
 
-        RoughDragCal(qubit, self.cals, backend=self.backend).run().block_for_results()
+        expdata = RoughDragCal(qubit, self.cals, backend=self.backend).run()
+        self.assertExperimentDone(expdata)
 
         new_beta = self.cals.get_parameter_value("β", (0,), "x")
         self.assertTrue(abs(new_beta - self.backend.ideal_beta) < self.test_tol)

test/calibration/experiments/test_fine_amplitude.py

@@ -51,6 +51,7 @@ def test_end_to_end_under_rotation(self, pi_ratio):
         backend = MockFineAmp(error, np.pi, "xp")
 
         expdata = amp_exp.run(backend)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
 
@@ -72,6 +73,7 @@ def test_end_to_end_over_rotation(self, pi_ratio):
         backend = MockFineAmp(error, np.pi, "xp")
 
         expdata = amp_exp.run(backend)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
 
@@ -215,6 +217,7 @@ def test_run_x_cal(self):
 
         # run the calibration experiment. This should update the amp parameter of x which we test.
         exp_data = amp_cal.run(self.backend)
+        self.assertExperimentDone(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         new_amp = init_amp * np.pi / (np.pi + d_theta)
 
@@ -253,6 +256,7 @@ def test_run_sx_cal(self):
 
         # run the calibration experiment. This should update the amp parameter of x which we test.
         exp_data = amp_cal.run(MockFineAmp(-np.pi * 0.07, np.pi / 2, "sx"))
+        self.assertExperimentDone(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         new_amp = init_amp * (np.pi / 2) / (np.pi / 2 + d_theta)
 

test/calibration/experiments/test_fine_drag.py

@@ -66,13 +66,15 @@ def test_end_to_end(self):
         drag.set_experiment_options(schedule=self.schedule)
         drag.set_transpile_options(basis_gates=["rz", "Drag", "sx"])
         exp_data = drag.run(FineDragTestBackend())
+        self.assertExperimentDone(exp_data)
 
         self.assertEqual(exp_data.analysis_results(0).quality, "good")
 
     def test_end_to_end_no_schedule(self):
         """Test that we can run without a schedule."""
 
         exp_data = FineXDrag(0).run(FineDragTestBackend())
+        self.assertExperimentDone(exp_data)
 
         self.assertEqual(exp_data.analysis_results(0).quality, "good")
 
@@ -127,6 +129,7 @@ def test_update_cals(self):
 
         # run the calibration experiment. This should update the beta parameter of x which we test.
         exp_data = drag_cal.run(self.backend)
+        self.assertExperimentDone(exp_data)
         d_theta = exp_data.analysis_results(1).value.value
         sigma = 40
         target_angle = np.pi

test/calibration/experiments/test_fine_frequency.py

@@ -55,7 +55,8 @@ def test_end_to_end(self, freq_shift):
         freq_exp = FineFrequency(0, 160, backend)
         freq_exp.set_transpile_options(inst_map=self.inst_map)
 
-        expdata = freq_exp.run(shots=100).block_for_results()
+        expdata = freq_exp.run(shots=100)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(1)
         d_theta = result.value.value
         dt = backend.configuration().dt
@@ -79,7 +80,8 @@ def test_calibration_version(self):
 
         self.assertAlmostEqual(freq_before, armonk_freq)
 
-        fine_freq.run().block_for_results()
+        expdata = fine_freq.run()
+        self.assertExperimentDone(expdata)
 
         freq_after = self.cals.get_parameter_value(self.cals.__drive_freq_parameter__, 0)
 

test/calibration/experiments/test_rabi.py

@@ -29,6 +29,7 @@
 from qiskit_experiments.data_processing.data_processor import DataProcessor
 from qiskit_experiments.data_processing.nodes import Probability
 from qiskit_experiments.test.mock_iq_backend import RabiBackend
+from qiskit_experiments.database_service.db_experiment_data import ExperimentStatus
 
 
 class TestRabiEndToEnd(QiskitExperimentsTestCase):
@@ -54,6 +55,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(0)
 
         self.assertEqual(result.quality, "good")
@@ -64,6 +66,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(0)
         self.assertEqual(result.quality, "good")
         self.assertTrue(abs(result.value.value[1] - backend.rabi_rate) < test_tol)
@@ -73,6 +76,7 @@ def test_rabi_end_to_end(self):
         rabi = Rabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 101))
         expdata = rabi.run(backend)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(0)
         self.assertEqual(result.quality, "good")
         self.assertTrue(abs(result.value.value[1] - backend.rabi_rate) < test_tol)
@@ -91,6 +95,7 @@ def test_wrong_processor(self):
         data = rabi.run(backend)
         result = data.analysis_results()
 
+        self.assertEqual(data.status(), ExperimentStatus.ERROR)
         self.assertEqual(len(result), 0)
 
     def test_experiment_config(self):
@@ -135,6 +140,7 @@ def test_ef_rabi_end_to_end(self):
         rabi = EFRabi(self.qubit, self.sched)
         rabi.set_experiment_options(amplitudes=np.linspace(-0.95, 0.95, 21))
         expdata = rabi.run(backend)
+        self.assertExperimentDone(expdata)
         result = expdata.analysis_results(1)
 
         self.assertEqual(result.quality, "good")

test/calibration/experiments/test_ramsey_xy.py

@@ -44,6 +44,7 @@ def test_end_to_end(self):
 
         for freq_shift in [2e6, -3e6]:
             test_data = ramsey.run(MockRamseyXY(freq_shift=freq_shift))
+            self.assertExperimentDone(test_data)
             meas_shift = test_data.analysis_results(1).value.value
             self.assertTrue((meas_shift - freq_shift) < abs(test_tol * freq_shift))
 
@@ -62,7 +63,8 @@ def test_update_calibrations(self):
         freq_shift = 4e6
         osc_shift = 2e6
         backend = MockRamseyXY(freq_shift=freq_shift + osc_shift)  # oscillation with 6 MHz
-        FrequencyCal(0, self.cals, backend, osc_freq=osc_shift).run().block_for_results()
+        expdata = FrequencyCal(0, self.cals, backend, osc_freq=osc_shift).run()
+        self.assertExperimentDone(expdata)
 
         # Check that qubit frequency after running the cal is shifted by freq_shift, i.e. 4 MHz.
         f01 = self.cals.get_parameter_value(freq_name, 0)

test/calibration/experiments/test_rough_amplitude.py

@@ -60,7 +60,8 @@ def test_update(self):
         self.assertTrue(np.allclose(self.cals.get_parameter_value("amp", 0, "sx"), 0.25))
 
         rabi_ef = RoughXSXAmplitudeCal(0, self.cals)
-        rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5)).block_for_results()
+        expdata = rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5))
+        self.assertExperimentDone(expdata)
 
         tol = 0.002
         self.assertTrue(abs(self.cals.get_parameter_value("amp", 0, "x") - 0.333) < tol)
@@ -135,7 +136,8 @@ def test_ef_update(self):
         self.assertTrue(np.allclose(self.cals.get_parameter_value("amp", 0, "sx12"), 0.2))
 
         rabi_ef = EFRoughXSXAmplitudeCal(0, self.cals)
-        rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5)).block_for_results()
+        expdata = rabi_ef.run(RabiBackend(amplitude_to_angle=np.pi * 1.5))
+        self.assertExperimentDone(expdata)
 
         tol = 0.002
         self.assertTrue(abs(self.cals.get_parameter_value("amp", 0, "x12") - 0.333) < tol)

test/calibration/experiments/test_rough_frequency.py

@@ -60,7 +60,8 @@ def test_update_calibrations(self):
 
         frequencies = np.linspace(freq01 - 10.0e6, freq01 + 10.0e6, 21)
 
-        RoughFrequencyCal(0, cals, frequencies).run(backend).block_for_results()
+        expdata = RoughFrequencyCal(0, cals, frequencies).run(backend)
+        self.assertExperimentDone(expdata)
 
         # Check the updated frequency which should be shifted by 5MHz.
         post_freq = cals.get_parameter_value(cals.__drive_freq_parameter__, (0,))

test/calibration/test_update_library.py

@@ -66,6 +66,7 @@ def test_frequency(self):
         spec = QubitSpectroscopy(qubit, frequencies)
         spec.set_run_options(meas_level=MeasLevel.CLASSIFIED)
         exp_data = spec.run(backend)
+        self.assertExperimentDone(exp_data)
         result = exp_data.analysis_results(1)
         value = result.value.value
 

test/database_service/test_db_experiment_data.py

@@ -129,9 +129,9 @@ def test_add_data_job(self):
 
         exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
         exp_data.add_jobs(a_job)
-        exp_data.block_for_results()
+        self.assertExperimentDone(exp_data)
         exp_data.add_jobs(jobs)
-        exp_data.block_for_results()
+        self.assertExperimentDone(exp_data)
         self.assertEqual(expected, [sdata["counts"] for sdata in exp_data.data()])
         self.assertIn(a_job.job_id(), exp_data.job_ids)
 
@@ -156,7 +156,7 @@ def _callback(_exp_data):
         exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
         exp_data.add_jobs(a_job)
         exp_data.add_analysis_callback(_callback)
-        exp_data.block_for_results()
+        self.assertExperimentDone(exp_data)
         self.assertTrue(called_back)
 
     def test_add_data_callback(self):
@@ -189,7 +189,7 @@ def _callback(_exp_data):
             with self.subTest(data=data):
                 exp_data.add_data(data)
                 exp_data.add_analysis_callback(_callback)
-                exp_data.block_for_results()
+                self.assertExperimentDone(exp_data)
 
         self.assertEqual(len(subtests), called_back_count)
 
@@ -211,7 +211,7 @@ def _callback(_exp_data, **kwargs):
         exp_data = DbExperimentData(backend=self.backend, experiment_type="qiskit_test")
         exp_data.add_jobs(a_job)
         exp_data.add_analysis_callback(_callback, foo=callback_kwargs)
-        exp_data.block_for_results()
+        self.assertExperimentDone(exp_data)
         self.assertTrue(called_back)
 
     def test_add_data_pending_post_processing(self):
@@ -637,8 +637,7 @@ def test_status_done(self):
         exp_data = DbExperimentData(experiment_type="qiskit_test")
         exp_data.add_jobs(job)
         exp_data.add_jobs(job)
-        exp_data.add_analysis_callback(lambda *args, **kwargs: time.sleep(1))
-        exp_data.block_for_results()
+        self.assertExperimentDone(exp_data)
         self.assertEqual(ExperimentStatus.DONE, exp_data.status())
 
     def test_set_tags(self):
@@ -667,7 +666,7 @@ def _job_result():
         # Cleanup
         with self.assertLogs("qiskit_experiments", "WARNING"):
             event.set()
-            exp_data.block_for_results()
+            self.assertExperimentDone(exp_data)
 
     def test_cancel_analysis(self):
         """Test canceling experiment analysis."""
@@ -813,7 +812,7 @@ def _sleeper(*args, **kwargs):  # pylint: disable=unused-argument
         exp_data = DbExperimentData(experiment_type="qiskit_test")
         exp_data.add_jobs(job)
         exp_data.add_analysis_callback(_sleeper)
-        exp_data.block_for_results()
+        self.assertExperimentDone(exp_data)
         self.assertEqual(2, sleep_count)
 
     def test_additional_attr(self):

test/quantum_volume/test_qv.py

@@ -105,8 +105,10 @@ def test_qv_sigma_decreasing(self):
         # set number of trials to a low number to make the test faster
         qv_exp.set_experiment_options(trials=2)
         expdata1 = qv_exp.run(backend)
+        self.assertExperimentDone(expdata1)
         result_data1 = expdata1.analysis_results(0)
         expdata2 = qv_exp.run(backend, analysis=None)
+        self.assertExperimentDone(expdata2)
         expdata2.add_data(expdata1.data())
         qv_exp.analysis.run(expdata2)
         result_data2 = expdata2.analysis_results(0)

test/randomized_benchmarking/test_rb.py

@@ -58,6 +58,7 @@ def test_rb_experiment(self, qubits: list):
             seed=exp_attributes["seed"],
         )
         exp_data = rb_exp.run(backend)
+        self.assertExperimentDone(exp_data)
         exp = exp_data.experiment
         exp_circuits = rb_exp.circuits()
         self.validate_metadata(exp_circuits, exp_attributes)
@@ -203,6 +204,7 @@ def test_interleaved_rb_experiment(self, interleaved_element: "Gate", qubits: li
             seed=exp_attributes["seed"],
         )
         experiment_obj = rb_exp.run(backend)
+        self.assertExperimentDone(experiment_obj)
         exp_data = experiment_obj.experiment
         exp_circuits = rb_exp.circuits()
         self.validate_metadata(exp_circuits, exp_attributes)

test/randomized_benchmarking/test_rb_analysis.py

@@ -205,7 +205,8 @@ def _load_rb_data(self, rb_exp_data_file_name: str):
             ((0, 1), "cx"): 1,
         }
         rb_exp.analysis.set_options(gate_error_ratio=gate_error_ratio)
-        analysis_results = rb_exp.analysis.run(expdata1).block_for_results()
+        analysis_results = rb_exp.analysis.run(expdata1)
+        self.assertExperimentDone(analysis_results)
         return data, analysis_results
 
 
@@ -260,7 +261,8 @@ def _load_rb_data(self, rb_exp_data_file_name: str):
             ((0, 1), "cx"): 1,
         }
         rb_exp.analysis.set_options(gate_error_ratio=gate_error_ratio)
-        analysis_results = rb_exp.analysis.run(expdata1).block_for_results()
+        analysis_results = rb_exp.analysis.run(expdata1)
+        self.assertExperimentDone(analysis_results)
         return data, analysis_results
 
     def test_interleaved_rb_analysis_test(self):