Version RunDescriber, and 5->6 DB upgrade, also reorg run descriptions module structure

.travis.yml

@@ -51,6 +51,7 @@ script:
       python generate_version_3.py
       python generate_version_4a.py
       python generate_version_4.py
+      python generate_version_5.py
       cd $TRAVIS_BUILD_DIR
       pip uninstall -y qcodes
       pip install .

azure-pipelines.yml

@@ -65,6 +65,7 @@ jobs:
           python generate_version_3.py
           python generate_version_4a.py
           python generate_version_4.py
+          python generate_version_5.py
         displayName: "Generate db fixtures"
         condition: succeededOrFailed()
       - script: |

docs/examples/DataSet/Dataset Context Manager.ipynb

@@ -50,7 +50,7 @@
     "                                                 load_or_create_experiment)\n",
     "from qcodes.dataset.database import initialise_database\n",
     "from qcodes.tests.instrument_mocks import DummyInstrument\n",
-    "from qcodes.dataset.param_spec import ParamSpec\n",
+    "from qcodes.dataset.descriptions.param_spec import ParamSpec\n",
     "from qcodes.dataset.measurements import Measurement\n",
     "from qcodes.dataset.plotting import plot_by_id\n",
     "from qcodes.dataset.data_export import get_shaped_data_by_runid\n",

docs/examples/DataSet/Pedestrian example of subscribing to a DataSet.ipynb

@@ -37,12 +37,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
     "from qcodes.dataset.database import initialise_database\n",
-    "from qcodes.dataset.data_set import new_data_set, ParamSpec\n",
+    "from qcodes.dataset.data_set import new_data_set\n",
+    "from qcodes.dataset.descriptions.param_spec import ParamSpec\n",
     "from qcodes.dataset.experiment_container import new_experiment\n",
     "import logging\n",
     "import warnings\n",

qcodes/__init__.py

@@ -80,7 +80,7 @@
 from qcodes.dataset.experiment_container import new_experiment, load_experiment, load_experiment_by_name, \
     load_last_experiment, experiments, load_or_create_experiment
 from qcodes.dataset.sqlite.settings import SQLiteSettings
-from qcodes.dataset.param_spec import ParamSpec
+from qcodes.dataset.descriptions.param_spec import ParamSpec
 from qcodes.dataset.sqlite.database import initialise_database, \
     initialise_or_create_database_at
 

qcodes/dataset/data_set.py

@@ -1,6 +1,6 @@
 import functools
 import json
-from typing import (Any, Dict, List, Optional, Union, Sized, Callable, cast,
+from typing import (Any, Dict, List, Optional, Union, Sized, Callable,
                     Sequence)
 from threading import Thread
 import time
@@ -11,7 +11,8 @@
 import numpy
 import pandas as pd
 
-from qcodes.dataset.param_spec import ParamSpec, ParamSpecBase
+from qcodes.dataset.descriptions.param_spec import ParamSpec, ParamSpecBase
+import qcodes.dataset.descriptions.versioning.serialization as serial
 from qcodes.dataset.sqlite.connection import atomic, atomic_transaction, \
     transaction, make_connection_plus_from, ConnectionPlus
 from qcodes.dataset.sqlite.queries import add_parameter, create_run, \
@@ -27,11 +28,12 @@
     insert_many_values, insert_values, VALUE, one
 from qcodes.dataset.sqlite.database import get_DB_location, connect
 from qcodes.instrument.parameter import _BaseParameter
-from qcodes.dataset.descriptions import RunDescriber
-from qcodes.dataset.dependencies import (InterDependencies,
-                                         InterDependencies_,
-                                         old_to_new, new_to_old,
-                                         DependencyError)
+from qcodes.dataset.descriptions.rundescriber import RunDescriber
+from qcodes.dataset.descriptions.dependencies import (InterDependencies_,
+                                                      DependencyError)
+from qcodes.dataset.descriptions.versioning.v0 import InterDependencies
+from qcodes.dataset.descriptions.versioning.converters import old_to_new, \
+    new_to_old
 from qcodes.dataset.guids import generate_guid
 from qcodes.utils.deprecate import deprecate
 import qcodes.config
@@ -247,15 +249,7 @@ def __init__(self, path_to_db: str = None,
                                  f"the database")
             self._completed = completed(self.conn, self.run_id)
             run_desc = self._get_run_description_from_db()
-            if run_desc._old_style_deps:
-                # TODO: what if the old run had invalid interdep.s?
-                old_idps: InterDependencies = cast(InterDependencies,
-                                                   run_desc.interdeps)
-                self._interdeps = old_to_new(old_idps)
-            else:
-                new_idps: InterDependencies_ = cast(InterDependencies_,
-                                                    run_desc.interdeps)
-                self._interdeps = new_idps
+            self._interdeps = run_desc.interdeps
             self._metadata = get_metadata_from_run_id(self.conn, run_id)
             self._started = self.run_timestamp_raw is not None
 
@@ -464,7 +458,7 @@ def _get_run_description_from_db(self) -> RunDescriber:
         Look up the run_description from the database
         """
         desc_str = get_run_description(self.conn, self.run_id)
-        return RunDescriber.from_json(desc_str)
+        return serial.read_json_to_current(desc_str)
 
     def toggle_debug(self):
         """
@@ -590,8 +584,9 @@ def _perform_start_actions(self) -> None:
         for spec in paramspecs:
             add_parameter(self.conn, self.table_name, spec)
 
-        update_run_description(self.conn, self.run_id,
-                               self.description.to_json())
+        desc_str = serial.make_json_for_storage(self.description)
+
+        update_run_description(self.conn, self.run_id, desc_str)
 
         set_run_timestamp(self.conn, self.run_id)
 
@@ -901,6 +896,7 @@ def get_setpoints(self, param_name: str) -> Dict[str, List[List[Any]]]:
         if param_name not in self.parameters:
             raise ValueError('Unknown parameter, not in this DataSet')
 
+
         if paramspec not in self._interdeps.dependencies.keys():
             raise ValueError(f'Parameter {param_name} has no setpoints.')
 

qcodes/dataset/database_extract_runs.py

@@ -4,7 +4,7 @@
 
 import numpy as np
 
-from qcodes.dataset.dependencies import new_to_old
+from qcodes.dataset.descriptions.versioning.converters import new_to_old
 from qcodes.dataset.data_set import DataSet
 from qcodes.dataset.experiment_container import load_or_create_experiment
 from qcodes.dataset.sqlite.connection import atomic, ConnectionPlus

qcodes/dataset/database_fix_functions.py

@@ -8,8 +8,8 @@
 
 from tqdm import tqdm
 
-from qcodes.dataset.descriptions import RunDescriber
-from qcodes.dataset.dependencies import InterDependencies
+import qcodes.dataset.descriptions.versioning.v0 as v0
+import qcodes.dataset.descriptions.versioning.serialization as serial
 from qcodes.dataset.sqlite.connection import ConnectionPlus, atomic, \
     atomic_transaction
 from qcodes.dataset.sqlite.db_upgrades import get_user_version
@@ -25,7 +25,7 @@ def fix_version_4a_run_description_bug(conn: ConnectionPlus) -> Dict[str, int]:
     """
     Fix function to fix a bug where the RunDescriber accidentally wrote itself
     to string using the (new) InterDependencies_ object instead of the (old)
-    InterDependencies object. After the first run, this function should be
+    InterDependencies object. After the first call, this function should be
     idempotent.
 
 
@@ -55,18 +55,32 @@ def fix_version_4a_run_description_bug(conn: ConnectionPlus) -> Dict[str, int]:
         runs_inspected = 0
         runs_fixed = 0
 
+        old_style_keys = ['paramspecs']
+        new_style_keys = ['parameters', 'dependencies', 'inferences',
+                          'standalones']
+
         for run_id in pbar:
 
             desc_str = get_run_description(conn, run_id)
             desc_ser = json.loads(desc_str)
             idps_ser = desc_ser['interdependencies']
 
-            if RunDescriber._is_description_old_style(idps_ser):
+            if list(idps_ser.keys()) == old_style_keys:
                 pass
-            else:
-                new_desc = RunDescriber.from_json(desc_str)
-                update_run_description(conn, run_id, new_desc.to_json())
+            elif list(idps_ser.keys()) == new_style_keys:
+                # we cheat a little here; we introduce a RunDescriber version
+                # number into a DB of schema version 4. That should be okay,
+                # since the only rule is that DBs of schema versions > 4
+                # MUST have a versioned RunDescriber,
+                desc_ser['version'] = 1
+                new_desc = serial.deserialize_to_current(desc_ser)
+                json_str = serial.make_json_in_version(new_desc, 0)
+                # the json_str has a "version" field with value 0
+                update_run_description(conn, run_id, json_str)
                 runs_fixed += 1
+            else:
+                raise RuntimeError(f'Invalid runs_description for run_id: '
+                                   f'{run_id}')
 
             runs_inspected += 1
 
@@ -98,17 +112,19 @@ def fix_wrong_run_descriptions(conn: ConnectionPlus,
     log.info('[*] Fixing run descriptions...')
     for run_id in run_ids:
         trusted_paramspecs = get_parameters(conn, run_id)
-        trusted_desc = RunDescriber(
-                           interdeps=InterDependencies(*trusted_paramspecs))
+        trusted_desc = v0.RunDescriber(
+            v0.InterDependencies(*trusted_paramspecs))
 
         actual_desc_str = select_one_where(conn, "runs",
                                            "run_description",
                                            "run_id", run_id)
 
-        if actual_desc_str == trusted_desc.to_json():
+        trusted_json = serial.make_json_in_version(trusted_desc, 0)
+
+        if actual_desc_str == trusted_json:
             log.info(f'[+] Run id: {run_id} had an OK description')
         else:
             log.info(f'[-] Run id: {run_id} had a broken description. '
                      f'Description found: {actual_desc_str}')
-            update_run_description(conn, run_id, trusted_desc.to_json())
+            update_run_description(conn, run_id, trusted_json)
             log.info(f'    Run id: {run_id} has been updated.')

qcodes/dataset/descriptions/__init__.py

@@ -0,0 +1,8 @@
+"""
+Module containing all objects used for the description of a run.
+
+Since run descriptions are persisted to disk, we have a strong obligation to
+ensure backwards compatibility. The code is organised such that everything at
+this module's top level is the CURRENT version. Older objects and the functions
+necessary to convert between different versions are found in .versioning
+"""