Enable remaining parameter tests (#86)

* Enable remaining parameter tests
* Include optimization parameter api layer (#89)
* Bump several dependency versions
* Let api/column handle both tables and parameters
* Make api-layer tests pass
* Include optimization parameter core layer (#90)
* Enable parameter core layer and test it
* Fix things after rebase
* Ensure all intended changes survive the rebase
* Adapt data validation function for parameters
* Allow tests to pass again

Author: glatterf42

ixmp4/core/optimization/data.py

@@ -2,6 +2,7 @@
 
 from ..base import BaseFacade
 from .indexset import IndexSetRepository
+from .parameter import ParameterRepository
 from .scalar import ScalarRepository
 from .table import TableRepository
 
@@ -11,11 +12,13 @@ class OptimizationData(BaseFacade):
     IndexSet, Table, Variable, etc."""
 
     indexsets: IndexSetRepository
+    parameters: ParameterRepository
     scalars: ScalarRepository
     tables: TableRepository
 
     def __init__(self, *args, run: Run, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.indexsets = IndexSetRepository(_backend=self.backend, _run=run)
+        self.parameters = ParameterRepository(_backend=self.backend, _run=run)
         self.scalars = ScalarRepository(_backend=self.backend, _run=run)
         self.tables = TableRepository(_backend=self.backend, _run=run)

ixmp4/core/optimization/parameter.py

@@ -34,12 +34,20 @@ def data(self) -> dict[str, Any]:
     def add(self, data: dict[str, Any] | pd.DataFrame) -> None:
         """Adds data to an existing Parameter."""
         self.backend.optimization.parameters.add_data(
-            table_id=self._model.id, data=data
+            parameter_id=self._model.id, data=data
         )
         self._model.data = self.backend.optimization.parameters.get(
             run_id=self._model.run__id, name=self._model.name
         ).data
 
+    @property
+    def values(self) -> list:
+        return self._model.data.get("values", [])
+
+    @property
+    def units(self) -> list:
+        return self._model.data.get("units", [])
+
     @property
     def constrained_to_indexsets(self) -> list[str]:
         return [column.indexset.name for column in self._model.columns]

ixmp4/data/abstract/optimization/column.py

@@ -14,8 +14,10 @@ class Column(base.BaseModel, Protocol):
     """Unique name of the Column."""
     dtype: types.String
     """Type of the Column's data."""
-    table__id: types.Integer
+    table__id: types.Mapped[int | None]
     """Foreign unique integer id of a Table."""
+    parameter__id: types.Mapped[int | None]
+    """Foreign unique integer id of a Parameter."""
     indexset: types.Mapped[IndexSet]
     """Associated IndexSet."""
     constrained_to_indexset: types.Integer

ixmp4/data/abstract/optimization/parameter.py

@@ -170,7 +170,7 @@ def tabulate(self, *, name: str | None = None, **kwargs) -> pd.DataFrame:
 
     # TODO Once present, state how to check which IndexSets are linked and which values
     # they permit
-    def add_data(self, table_id: int, data: dict[str, Any] | pd.DataFrame) -> None:
+    def add_data(self, parameter_id: int, data: dict[str, Any] | pd.DataFrame) -> None:
         r"""Adds data to a Parameter.
 
         The data will be validated with the linked constrained
@@ -183,7 +183,7 @@ def add_data(self, table_id: int, data: dict[str, Any] | pd.DataFrame) -> None:
 
         Parameters
         ----------
-        table_id : int
+        parameter_id : int
             The id of the :class:`ixmp4.data.abstract.optimization.Parameter`.
         data : dict[str, Any] | pandas.DataFrame
             The data to be added.

ixmp4/data/api/optimization/column.py

@@ -14,7 +14,8 @@ class Column(base.BaseModel):
     id: int
     name: str
     dtype: str
-    table__id: int
+    table__id: int | None
+    parameter__id: int | None
     indexset: IndexSet
     constrained_to_indexset: int
     unique: bool

ixmp4/data/api/optimization/parameter.py

@@ -57,13 +57,13 @@ def create(
             column_names=column_names,
         )
 
-    def add_data(self, table_id: int, data: dict[str, Any] | pd.DataFrame) -> None:
+    def add_data(self, parameter_id: int, data: dict[str, Any] | pd.DataFrame) -> None:
         if isinstance(data, pd.DataFrame):
             # data will always contains str, not only Hashable
             data: dict[str, Any] = data.to_dict(orient="list")  # type: ignore
         kwargs = {"data": data}
         self._request(
-            method="PATCH", path=self.prefix + str(table_id) + "/data/", json=kwargs
+            method="PATCH", path=self.prefix + str(parameter_id) + "/data/", json=kwargs
         )
 
     def get(self, run_id: int, name: str) -> Parameter:

ixmp4/data/db/iamc/base.py

@@ -7,7 +7,6 @@
     Deleter,
     Enumerator,
     Lister,
-    NameMixin,
     Retriever,
     Selecter,
     Tabulator,

ixmp4/data/db/optimization/base.py

@@ -8,18 +8,14 @@
     Deleter,
     Enumerator,
     Lister,
-    OptimizationDataMixin,
-    OptimizationNameMixin,
     Retriever,
-    RunIDMixin,
     Selecter,
     Tabulator,
     TimestampMixin,
-    UniqueNameRunIDMixin,
 )
 
 
-class BaseModel(RootBaseModel, OptimizationNameMixin, TimestampMixin):
+class BaseModel(RootBaseModel, TimestampMixin):
     __abstract__ = True
     table_prefix = "optimization_"
 

ixmp4/data/db/optimization/column/model.py

@@ -1,7 +1,5 @@
 from typing import ClassVar
 
-from sqlalchemy import UniqueConstraint
-
 from ixmp4 import db
 from ixmp4.data import types
 from ixmp4.data.abstract import optimization as abstract
@@ -34,4 +32,4 @@ class Column(base.BaseModel):
     # Currently not in use:
     unique: types.Boolean = db.Column(db.Boolean, default=True)
 
-    __table_args__ = (UniqueConstraint("name", "table__id", "parameter__id"),)
+    __table_args__ = (db.UniqueConstraint("name", "table__id"),)

ixmp4/data/db/optimization/indexset/model.py

@@ -9,7 +9,7 @@
 from .. import base
 
 
-class IndexSet(base.BaseModel, base.UniqueNameRunIDMixin):
+class IndexSet(base.BaseModel):
     NotFound: ClassVar = abstract.IndexSet.NotFound
     NotUnique: ClassVar = abstract.IndexSet.NotUnique
     DeletionPrevented: ClassVar = abstract.IndexSet.DeletionPrevented

ixmp4/data/db/optimization/parameter/model.py

@@ -1,30 +1,37 @@
+import copy
 from typing import Any, ClassVar
 
-import pandas as pd
 from sqlalchemy.orm import validates
 
 from ixmp4 import db
 from ixmp4.data import types
 from ixmp4.data.abstract import optimization as abstract
 
-from .. import Column, base
+from .. import Column, base, utils
 
 
-class Parameter(base.BaseModel, base.OptimizationDataMixin, base.UniqueNameRunIDMixin):
+class Parameter(base.BaseModel):
     # NOTE: These might be mixin-able, but would require some abstraction
     NotFound: ClassVar = abstract.Parameter.NotFound
     NotUnique: ClassVar = abstract.Parameter.NotUnique
     DeletionPrevented: ClassVar = abstract.Parameter.DeletionPrevented
 
     # constrained_to_indexsets: ClassVar[list[str] | None] = None
 
-    # TODO Same as in table/model.py
-    columns: types.Mapped[list["Column"]] = db.relationship()  # type: ignore
+    run__id: types.RunId
+    columns: types.Mapped[list["Column"]] = db.relationship()
+    data: types.JsonDict = db.Column(db.JsonType, nullable=False, default={})
 
     @validates("data")
     def validate_data(self, key, data: dict[str, Any]):
-        data_frame: pd.DataFrame = pd.DataFrame.from_dict(data)
-        data_frame_to_validate = data_frame.drop(columns=["values", "units"])
-
-        self._validate_data(data_frame=data_frame_to_validate, data=data)
-        return data_frame.to_dict(orient="list")
+        data_to_validate = copy.deepcopy(data)
+        del data_to_validate["values"]
+        del data_to_validate["units"]
+        _ = utils.validate_data(
+            key=key,
+            data=data_to_validate,
+            columns=self.columns,
+        )
+        return data
+
+    __table_args__ = (db.UniqueConstraint("name", "run__id"),)

ixmp4/data/db/optimization/parameter/repository.py

@@ -74,7 +74,8 @@ def add(
         run_id: int,
         name: str,
     ) -> Parameter:
-        parameter = Parameter(name=name, run__id=run_id, **self.get_creation_info())
+        parameter = Parameter(name=name, run__id=run_id)
+        parameter.set_creation_info(auth_context=self.backend.auth_context)
         self.session.add(parameter)
 
         return parameter
@@ -154,7 +155,7 @@ def add_data(self, parameter_id: int, data: dict[str, Any] | pd.DataFrame) -> No
         missing_columns = set(["values", "units"]) - set(data.columns)
         assert (
             not missing_columns
-        ), f"Parameter.data must include the column(s): {' ,'.join(missing_columns)}!"
+        ), f"Parameter.data must include the column(s): {', '.join(missing_columns)}!"
 
         # Can use a set for now, need full column if we care about order
         for unit_name in set(data["units"]):