Merge pull request #5289 from jenshnielsen/opentelemetry_exp

Instrument Measurement and dond with opentelemetry

docs/changes/newsfragments/5289.new

@@ -0,0 +1,7 @@
+The QCoDeS Measurement Context manager, DataSaverBuilder and DoND functions have been instrumented as OpenTelemetry traces.
+This enables users to correlate log messages with the the measurement that they were performed within.
+See the `OpenTelemetry <https://opentelemetry.io/>`_ documentation for examples of how this can be used.
+
+The log exporting using OpenCensus within QCoDeS is expected to be deprecated and eventually removed in line
+with OpenCensus being discontinued. Users that are interested in gathering telemetry of their setups are encouraged
+to provide their own solution based on OpenTelemetry.

docs/examples/logging/logging_example.ipynb

@@ -1336,12 +1336,31 @@
     "For an analysis of the timestamps please also refer to the log analysis [example notebook](logfile_parsing.ipynb)."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Telemetry"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The QCoDeS Measurement Context manager, DataSaverBuilder and DoND functions are instrumented as OpenTelemetry traces. This enables users to correlate log messages with the the measurement that they were performed within. See the (OpenTelemetry)[https://opentelemetry.io/] documentation for examples of how this can be used. "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
   }
  ],
  "metadata": {

pyproject.toml

@@ -46,6 +46,7 @@ dependencies = [
     "websockets>=9.1",
     "wrapt>=1.13.2",
     "xarray>=0.18.0",
+    "opentelemetry-api>=1.15.0",
     # transitive dependencies. We list these explicitly to",
     # ensure that we always use versions that do not have",
     # known security vulnerabilities",

qcodes/dataset/dond/do_0d.py

@@ -3,6 +3,8 @@
 import logging
 from typing import TYPE_CHECKING, cast
 
+from opentelemetry import trace
+
 from qcodes import config
 from qcodes.parameters import ParameterBase
 
@@ -13,11 +15,13 @@
 from .do_nd_utils import _handle_plotting, _register_parameters, _set_write_period
 
 LOG = logging.getLogger(__name__)
+TRACER = trace.get_tracer(__name__)
 
 if TYPE_CHECKING:
     from ..descriptions.versioning.rundescribertypes import Shapes
     from .do_nd_utils import AxesTupleListWithDataSet, ParamMeasT
 
+@TRACER.start_as_current_span("qcodes.dataset.do0d")
 def do0d(
     *param_meas: ParamMeasT,
     write_period: float | None = None,

qcodes/dataset/dond/do_1d.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING, cast
 
 import numpy as np
+from opentelemetry import trace
 from tqdm.auto import tqdm
 
 from qcodes import config
@@ -29,6 +30,7 @@
 from qcodes.parameters import ParameterBase
 
 LOG = logging.getLogger(__name__)
+TRACER = trace.get_tracer(__name__)
 
 if TYPE_CHECKING:
     from qcodes.dataset.descriptions.versioning.rundescribertypes import Shapes
@@ -40,6 +42,7 @@
     )
 
 
+@TRACER.start_as_current_span("qcodes.dataset.do1d")
 def do1d(
     param_set: ParameterBase,
     start: float,

qcodes/dataset/dond/do_2d.py

@@ -7,6 +7,7 @@
 from typing import TYPE_CHECKING, cast
 
 import numpy as np
+from opentelemetry import trace
 from tqdm.auto import tqdm
 
 from qcodes import config
@@ -29,6 +30,8 @@
 from qcodes.parameters import ParameterBase
 
 LOG = logging.getLogger(__name__)
+TRACER = trace.get_tracer(__name__)
+
 
 if TYPE_CHECKING:
     from qcodes.dataset.descriptions.versioning.rundescribertypes import Shapes
@@ -39,7 +42,7 @@
         ParamMeasT,
     )
 
-
+@TRACER.start_as_current_span("qcodes.dataset.do2d")
 def do2d(
     param_set1: ParameterBase,
     start1: float,

qcodes/dataset/dond/do_nd.py

@@ -9,6 +9,7 @@
 from typing import TYPE_CHECKING, Any, Callable, Union, cast
 
 import numpy as np
+from opentelemetry import trace
 from tqdm.auto import tqdm
 from typing_extensions import TypedDict
 
@@ -48,6 +49,8 @@
 
 SweepVarType = Any
 
+TRACER = trace.get_tracer(__name__)
+
 
 class ParameterGroup(TypedDict):
     params: tuple[ParamMeasT, ...]
@@ -565,6 +568,7 @@ def parameters(self) -> tuple[ParameterBase, ...]:
         return self._parameters
 
 
+@TRACER.start_as_current_span("qcodes.dataset.dond")
 def dond(
     *params: AbstractSweep | TogetherSweep | ParamMeasT | Sequence[ParamMeasT],
     write_period: float | None = None,

qcodes/dataset/measurement_extensions.py

@@ -6,6 +6,8 @@
 from dataclasses import dataclass
 from typing import Any, Callable
 
+from opentelemetry import trace
+
 from qcodes.dataset.dond.do_nd import _Sweeper
 from qcodes.dataset.dond.do_nd_utils import ParamMeasT, catch_interrupts
 from qcodes.dataset.dond.sweeps import AbstractSweep, LinSweep, TogetherSweep
@@ -14,6 +16,8 @@
 from qcodes.dataset.threading import process_params_meas
 from qcodes.parameters.parameter_base import ParameterBase
 
+TRACER = trace.get_tracer(__name__)
+
 
 @dataclass
 class DataSetDefinition:
@@ -85,12 +89,16 @@ def datasaver_builder(
     Yields:
         A list of generated datasavers with parameters registered
     """
+
     measurement_instances = setup_measurement_instances(
         dataset_definitions, override_experiment
     )
-    with catch_interrupts() as _, ExitStack() as stack:
+    with TRACER.start_as_current_span(
+        "qcodes.dataset.datasaver_builder"
+    ), catch_interrupts() as _, ExitStack() as stack:
+        datasaver_builder_span = trace.get_current_span()
         datasavers = [
-            stack.enter_context(measurement.run())
+            stack.enter_context(measurement.run(parent_span=datasaver_builder_span))
             for measurement in measurement_instances
         ]
         yield datasavers
@@ -153,31 +161,40 @@ def dond_into(
         additional_setpoints: A list of setpoint parameters to be registered in the
             measurement but not scanned/swept-over.
     """
-    sweep_instances, params_meas = parse_dond_into_args(*params)
-    sweeper = _Sweeper(sweep_instances, additional_setpoints)
-    for set_events in sweeper:
-        results: dict[ParameterBase, Any] = {}
-        additional_setpoints_data = process_params_meas(additional_setpoints)
-        for set_event in set_events:
-            if set_event.should_set:
-                set_event.parameter(set_event.new_value)
-                for act in set_event.actions:
-                    act()
-                time.sleep(set_event.delay)
-
-            if set_event.get_after_set:
-                results[set_event.parameter] = set_event.parameter()
-            else:
-                results[set_event.parameter] = set_event.new_value
-
-        meas_value_pair = process_params_meas(params_meas)
-        for meas_param, value in meas_value_pair:
-            results[meas_param] = value
-
-        datasaver.add_result(
-            *list(results.items()),
-            *additional_setpoints_data,
-        )
+    # at this stage multiple measurement context managers may be in run state
+    # as datasavers. Here we ensure we bind the parent span to the correct
+    # datasaver.
+    if datasaver._span is not None:
+        context = trace.set_span_in_context(datasaver._span)
+    else:
+        context = None
+
+    with TRACER.start_as_current_span("qcodes.dataset.dond_into", context=context):
+        sweep_instances, params_meas = parse_dond_into_args(*params)
+        sweeper = _Sweeper(sweep_instances, additional_setpoints)
+        for set_events in sweeper:
+            results: dict[ParameterBase, Any] = {}
+            additional_setpoints_data = process_params_meas(additional_setpoints)
+            for set_event in set_events:
+                if set_event.should_set:
+                    set_event.parameter(set_event.new_value)
+                    for act in set_event.actions:
+                        act()
+                    time.sleep(set_event.delay)
+
+                if set_event.get_after_set:
+                    results[set_event.parameter] = set_event.parameter()
+                else:
+                    results[set_event.parameter] = set_event.new_value
+
+            meas_value_pair = process_params_meas(params_meas)
+            for meas_param, value in meas_value_pair:
+                results[meas_param] = value
+
+            datasaver.add_result(
+                *list(results.items()),
+                *additional_setpoints_data,
+            )
 
 
 class LinSweeper(LinSweep):