Merge pull request #5294 from samantha-ho/samanthaho/complex_context

Add datasaver_builder and dond_core

docs/changes/newsfragments/5294.new

@@ -0,0 +1,3 @@
+New features: datasaver_builder and dond_into are intermediate measurement extensions designed to fill a gap between
+the low-level Measurement object and the high-level doNd functions. They allow convenient specification of parameter
+dependencies for registration and doNd-like syntax while also allowing direct access to the underlying datasaver objects.

docs/examples/DataSet/Accessing-data-in-DataSet.ipynb

@@ -18113,6 +18113,7 @@
     "    standalones = all_independents.difference(used_independents)\n",
     "    return tuple(ps for ps in paramspecs if ps.name in standalones)\n",
     "\n",
+    "\n",
     "all_parameters = dataset.get_parameters()\n",
     "standalone_parameters = get_standalone_parameters(all_parameters)\n",
     "standalone_parameters"

docs/examples/DataSet/Cache/write_for_caching.ipynb

@@ -149,7 +149,6 @@
     "        datasaver.flush_data_to_database(block=True)\n",
     "        time.sleep(0.1)\n",
     "\n",
-    "    \n",
     "    dataset = datasaver.dataset  # convenient to have for plotting"
    ]
   },

docs/examples/DataSet/Offline Plotting Tutorial.ipynb

@@ -649,18 +649,22 @@
     "tvals = np.linspace(-500, 1500, 25)\n",
     "\n",
     "# define two small forbidden range functions\n",
+    "\n",
+    "\n",
     "def no_x(xv):\n",
     "    if xv > 0 and xv < 3:\n",
     "        return True\n",
     "    else:\n",
     "        return False\n",
     "    \n",
+    "\n",
     "def no_t(tv):\n",
     "    if tv > 0 and tv < 450:\n",
     "        return True\n",
     "    else:\n",
     "        return False\n",
     "\n",
+    "\n",
     "with meas.run() as datasaver:\n",
     "    for xv in xvals:\n",
     "        for tv in tvals:\n",

docs/examples/DataSet/Offline plotting with categorical data.ipynb

@@ -151,7 +151,6 @@
     "        datasaver.add_result((fridge_config, configuration),\n",
     "                             (voltage, np.random.rand()))\n",
     "\n",
-    "\n",
     "    datasaver.add_result((fridge_config, 'open'),\n",
     "                         (voltage, np.random.rand()))\n",
     "        \n",
@@ -554,6 +553,7 @@
     "        return 'Possible'\n",
     "    return 'Good'\n",
     "\n",
+    "\n",
     "with meas.run() as datasaver:\n",
     "    \n",
     "    for bias_v in np.linspace(0, 3, 100):\n",
@@ -644,6 +644,7 @@
     "        return 'Possible'\n",
     "    return 'Good'\n",
     "\n",
+    "\n",
     "with meas.run() as datasaver:\n",
     "    \n",
     "    for bias_v in 3*(np.random.rand(100)):\n",
@@ -724,6 +725,7 @@
     "\n",
     "# a function to simulate the usefulness of a region\n",
     "\n",
+    "\n",
     "def get_usefulness(x, y):\n",
     "    x_num = samples.index(x)*4/len(samples)\n",
     "    val = np.sin(x_num)*np.sin(y)\n",
@@ -735,6 +737,7 @@
     "        return 'Possible'\n",
     "    return 'Good'\n",
     "\n",
+    "\n",
     "with meas.run() as datasaver:\n",
     "    \n",
     "    for samp in samples:\n",
@@ -810,6 +813,7 @@
     "\n",
     "# a function to simulate the usefulness of a region\n",
     "\n",
+    "\n",
     "def get_usefulness(x, y):\n",
     "    x_num = samples.index(x)*4/len(samples)\n",
     "    val = np.sin(x_num)*np.sin(y)\n",
@@ -821,6 +825,7 @@
     "        return 'Possible'\n",
     "    return 'Good'\n",
     "\n",
+    "\n",
     "with meas.run() as datasaver:\n",
     "    \n",
     "    for samp in samples:\n",

docs/examples/DataSet/Paramtypes explained.ipynb

@@ -106,6 +106,7 @@
    "source": [
     "# some array-like data types\n",
     "\n",
+    "\n",
     "class Spectrum(ArrayParameter):\n",
     "    \n",
     "    def __init__(self, name, instrument):\n",
@@ -150,13 +151,16 @@
     "                         setpoint_names=('Frequency0', 'Frequency1',\n",
     "                                         'Frequency2'),\n",
     "                             setpoint_units=('Hz', 'Other Hz', \"Third Hz\"))\n",
+    "\n",
     "    def get_raw(self):\n",
     "        a = self.npts[0]\n",
     "        b = self.npts[1]\n",
     "        c = self.npts[2]\n",
     "        return np.reshape(np.arange(a*b*c), (a, b, c))\n",
     "    \n",
     "# a string-valued parameter\n",
+    "\n",
+    "\n",
     "def dac1_too_high():\n",
     "    return 'Too high' if dac.ch1() > 5 else 'OK'"
    ]

docs/examples/DataSet/Performing-measurements-using-qcodes-parameters-and-dataset.ipynb

@@ -116,12 +116,15 @@
    "source": [
     "# now make some silly set-up and tear-down actions\n",
     "\n",
+    "\n",
     "def veryfirst():\n",
     "    print('Starting the measurement')\n",
     "\n",
+    "\n",
     "def numbertwo(inst1, inst2):\n",
     "    print(f'Doing stuff with the following two instruments: {inst1}, {inst2}')\n",
     "    \n",
+    "\n",
     "def thelast():\n",
     "    print('End of experiment')"
    ]
@@ -3317,6 +3320,7 @@
     "    dac.ch2(xk[1])\n",
     "    return dmm.v2.get()\n",
     "\n",
+    "\n",
     "noise = 0.0005\n",
     "x0 = [np.random.rand(), np.random.rand()]\n",
     "\n",

docs/examples/DataSet/Real_instruments/Example Measurements with Real Instruments.ipynb

@@ -917,14 +917,18 @@
    ],
    "source": [
     "# set-up and tear-down actions\n",
+    "\n",
+    "\n",
     "def setup():\n",
     "    key.ch1.output('ON')\n",
     "\n",
+    "\n",
     "def teardown():\n",
     "    key.ch1.output('OFF')\n",
     "\n",
     "# simple frequency sweep\n",
     "\n",
+    "\n",
     "meas = Measurement()\n",
     "meas.register_parameter(key.ch1.frequency)\n",
     "meas.register_parameter(sr830.R, setpoints=(key.ch1.frequency,))\n",
@@ -2636,6 +2640,8 @@
    "outputs": [],
    "source": [
     "# Prepare the scope\n",
+    "\n",
+    "\n",
     "def scope_setup():\n",
     "    mso.ch1.input('DC50')\n",
     "\n",
@@ -2665,6 +2671,7 @@
     "    \n",
     "    mso.write('run')\n",
     "\n",
+    "\n",
     "def keysight_setup():\n",
     "    # Prepare the Keysight\n",
     "    key.ch2.function_type('TRI')\n",
@@ -2673,6 +2680,7 @@
     "    key.ch2.burst_state('OFF')\n",
     "    key.ch2.output('ON')\n",
     "    \n",
+    "\n",
     "def keysight_teardown():\n",
     "    key.ch2.output('OFF')"
    ]

docs/examples/DataSet/Saving_data_in_the_background.ipynb

@@ -105,6 +105,7 @@
     "def gaussian(x, mu, sig):\n",
     "    return np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))\n",
     "\n",
+    "\n",
     "def get_response():\n",
     "    \"\"\"\n",
     "    Simulated slow instrument response\n",

docs/examples/Parameters/Legacy_Parameters.ipynb

@@ -58,6 +58,7 @@
     "        self._val += 6\n",
     "        return out\n",
     "\n",
+    "\n",
     "array_counter = ArrayCounter()\n",
     "\n",
     "# simple get\n",

docs/examples/Parameters/MultiParameter.ipynb

@@ -55,6 +55,7 @@
     "        scale_val = self._scale_param()\n",
     "        return (scale_val, scale_val / 2)\n",
     "\n",
+    "\n",
     "scale = ManualParameter('scale', initial_value=2)\n",
     "iq = SingleIQPair(scale_param=scale)\n",
     "\n",
@@ -93,6 +94,7 @@
     "        scale_val = self._scale_param()\n",
     "        return (self._indices * scale_val, self._indices * scale_val / 2)\n",
     "\n",
+    "\n",
     "iq_array = IQArray(scale_param=scale)\n",
     "\n",
     "# simple get\n",

docs/examples/Parameters/Parameter-With-Setpoints-defined-on-a-different-instrument.ipynb

@@ -90,6 +90,7 @@
     "    A parameter that generates a setpoint array from start, stop and num points\n",
     "    parameters.\n",
     "    \"\"\"\n",
+    "\n",
     "    def __init__(self, startparam, stopparam, numpointsparam, *args, **kwargs):\n",
     "        super().__init__(*args, **kwargs)\n",
     "        self._startparam = startparam\n",
@@ -101,7 +102,6 @@
     "                              self._numpointsparam())\n",
     "\n",
     "\n",
-    "    \n",
     "class DummyArray(ParameterWithSetpoints):\n",
     "    \n",
     "    def get_raw(self):\n",
@@ -115,7 +115,6 @@
     "        \n",
     "        super().__init__(name, **kwargs)\n",
     "            \n",
-    "\n",
     "        self.add_parameter('sweep_start',\n",
     "                           source=None,\n",
     "                           parameter_class=DelegateParameter)\n",

docs/examples/Parameters/Simple-Example-of-ParameterWithSetpoints.ipynb

@@ -92,6 +92,7 @@
     "    A parameter that generates a setpoint array from start, stop and num points\n",
     "    parameters.\n",
     "    \"\"\"\n",
+    "\n",
     "    def __init__(self, startparam, stopparam, numpointsparam, *args, **kwargs):\n",
     "        super().__init__(*args, **kwargs)\n",
     "        self._startparam = startparam\n",
@@ -102,6 +103,7 @@
     "        return np.linspace(self._startparam(), self._stopparam(),\n",
     "                              self._numpointsparam())\n",
     "\n",
+    "\n",
     "class DummyArray(ParameterWithSetpoints):\n",
     "    \n",
     "    def get_raw(self):\n",
@@ -115,7 +117,6 @@
     "        \n",
     "        super().__init__(name, **kwargs)\n",
     "            \n",
-    "\n",
     "        self.add_parameter('f_start',\n",
     "                           initial_value=0,\n",
     "                           unit='Hz',\n",

docs/examples/driver_examples/Qcodes example with HP8753D.ipynb

@@ -231,6 +231,8 @@
    "source": [
     "n_avgs = 10\n",
     "vna.number_of_averages(n_avgs)\n",
+    "\n",
+    "\n",
     "def run_sweeper():\n",
     "    vna.run_N_times(n_avgs)"
    ]

docs/examples/driver_examples/Qcodes example with Oxford Mercury iPS.ipynb

@@ -85,6 +85,7 @@
     "    \"\"\"\n",
     "    return np.sqrt(x**2 + y**2 + z**2) <= 1\n",
     "\n",
+    "\n",
     "# assign the limit function (this can also be done at init)\n",
     "mips.set_new_field_limits(spherical_limit)"
    ]

qcodes/dataset/__init__.py

@@ -35,6 +35,12 @@
 from .export_config import get_data_export_path
 from .guid_helpers import guids_from_dbs, guids_from_dir, guids_from_list_str
 from .legacy_import import import_dat_file
+from .measurement_extensions import (
+    DataSetDefinition,
+    LinSweeper,
+    datasaver_builder,
+    dond_into,
+)
 from .measurements import Measurement
 from .plotting import plot_by_id, plot_dataset
 from .sqlite.connection import ConnectionPlus
@@ -71,10 +77,13 @@
     "TogetherSweep",
     "call_params_threaded",
     "connect",
+    "datasaver_builder",
+    "DataSetDefinition",
     "do0d",
     "do1d",
     "do2d",
     "dond",
+    "dond_into",
     "experiments",
     "extract_runs_into_db",
     "get_data_export_path",
@@ -87,6 +96,7 @@
     "initialise_database",
     "initialise_or_create_database_at",
     "initialised_database_at",
+    "LinSweeper",
     "load_by_counter",
     "load_by_guid",
     "load_by_id",

qcodes/dataset/dond/do_1d.py

@@ -13,11 +13,11 @@
 from qcodes.dataset.descriptions.detect_shapes import detect_shape_of_measurement
 from qcodes.dataset.dond.do_nd_utils import (
     BreakConditionInterrupt,
-    _catch_interrupts,
     _handle_plotting,
     _register_actions,
     _register_parameters,
     _set_write_period,
+    catch_interrupts,
 )
 from qcodes.dataset.experiment_container import Experiment
 from qcodes.dataset.measurements import Measurement
@@ -146,7 +146,7 @@ def do1d(
     # do1D enforces a simple relationship between measured parameters
     # and set parameters. For anything more complicated this should be
     # reimplemented from scratch
-    with _catch_interrupts() as interrupted, meas.run() as datasaver, param_meas_caller as call_param_meas:
+    with catch_interrupts() as interrupted, meas.run() as datasaver, param_meas_caller as call_param_meas:
         dataset = datasaver.dataset
         additional_setpoints_data = process_params_meas(additional_setpoints)
         setpoints = np.linspace(start, stop, num_points)

qcodes/dataset/dond/do_2d.py

@@ -13,11 +13,11 @@
 from qcodes.dataset.descriptions.detect_shapes import detect_shape_of_measurement
 from qcodes.dataset.dond.do_nd_utils import (
     BreakConditionInterrupt,
-    _catch_interrupts,
     _handle_plotting,
     _register_actions,
     _register_parameters,
     _set_write_period,
+    catch_interrupts,
 )
 from qcodes.dataset.experiment_container import Experiment
 from qcodes.dataset.measurements import Measurement
@@ -39,6 +39,7 @@
         ParamMeasT,
     )
 
+
 def do2d(
     param_set1: ParameterBase,
     start1: float,
@@ -166,7 +167,7 @@ def do2d(
         else SequentialParamsCaller(*param_meas)
     )
 
-    with _catch_interrupts() as interrupted, meas.run() as datasaver, param_meas_caller as call_param_meas:
+    with catch_interrupts() as interrupted, meas.run() as datasaver, param_meas_caller as call_param_meas:
         dataset = datasaver.dataset
         additional_setpoints_data = process_params_meas(additional_setpoints)
         setpoints1 = np.linspace(start1, stop1, num_points1)