Avoid calculating non-sensical runtime profiles

When calculating runtime profiles, make sure we don't aggregate over multiple problem sizes (both objective and variables). Instead, raises an exception.

Partial fix for #4.

src/cocoviz/exceptions.py

@@ -7,4 +7,10 @@ class NoSuchIndicatorException(Exception):
 class IndicatorMismatchException(Exception):
     """Raised when indicators don't match between results"""
 
-    pass
+    pass
+
+
+class BadResultSetException(Exception):
+    """Raised when a runtime profile doesn't make sense for a result set"""    
+
+    pass

src/cocoviz/rtp.py

@@ -6,6 +6,7 @@
 
 from .targets import linear_targets
 from .types import ResultSet
+from .exceptions import BadResultSetException
 
 
 def runtime_profiles(
@@ -15,7 +16,7 @@ def runtime_profiles(
     number_of_targets: int = 101,
     targets: dict = None,
 ):
-    """Compute runtime profile for each algorithm in `results`.
+    """Compute a runtime profile for each algorithm in `results`.
 
     Parameters
     ----------
@@ -36,6 +37,13 @@ def runtime_profiles(
     dict
         Quantiles and probabilities for each algorithm in `results`.
     """
+
+    if len(results.number_of_variables) > 1:
+        raise BadResultSetException("Cannot derive runtime profile for problems with different number of variables.")
+
+    if len(results.number_of_objectives) > 1:
+        raise BadResultSetException("Cannot derive runtime profile for problems with different number of objectives.")
+
     # If no targets are given, calculate `number_of_targets` linearly spaced targets
     if not targets:
         targets = linear_targets(results, indicator, number_of_targets)

src/cocoviz/types.py

@@ -208,6 +208,8 @@ def append(self, result: Result) -> ResultSet:
             raise IndicatorMismatchException("Indicators in results don't match: {self._results[0].indicators} vs {result.indicators}")
         self.algorithms.add(result.algorithm)
         self.problems.add(result.problem)
+        self.number_of_variables.add(result.problem.number_of_variables)
+        self.number_of_objectives.add(result.problem.number_of_objectives)
         self._results.append(result)
         return self
 

tests/test_runtime_profile.py

@@ -0,0 +1,39 @@
+import pytest
+
+from cocoviz.exceptions import BadResultSetException
+from cocoviz import runtime_profiles, Result, ResultSet, ProblemDescription
+
+
+HV_a1 = {
+    "fevals":      [  1,   2,   3,  10,  20,  50, 100],
+    "hypervolume": [ 10,  20,  30,  40,  50,  60,  70],
+    "r2":          [ 20,  40,  60,  80, 100, 120, 140]
+}
+
+HV_a2 = {
+    "fevals":      [  1,   2,   3,  10,  20,  50, 100],
+    "hypervolume": [ 12,  22,  32,  42,  52,  62,  72],
+    "r2":          [ 22,  42,  62,  82, 102, 122, 142]
+}
+
+def test_aggregate_over_objectives():
+    pd_f1_d2 = ProblemDescription("f1", "i1", 10, 2)
+    pd_f1_d3 = ProblemDescription("f1", "i1", 10, 3)
+    rs = ResultSet([
+        Result("a1", pd_f1_d2, HV_a1),
+        Result("a1", pd_f1_d3, HV_a1)
+    ])
+
+    with pytest.raises(BadResultSetException):
+        runtime_profiles(rs, indicator="hypervolume")
+
+def test_aggregate_over_variables():
+    pd_f1_d2 = ProblemDescription("f1", "i1", 10, 2)
+    pd_f1_d3 = ProblemDescription("f1", "i1", 20, 2)
+    rs = ResultSet([
+        Result("a1", pd_f1_d2, HV_a1),
+        Result("a1", pd_f1_d3, HV_a1)
+    ])
+
+    with pytest.raises(BadResultSetException):
+        runtime_profiles(rs, indicator="hypervolume")