Add extra type for performance indicators

Instead of using strings to describe performance indicators, use a class that stores if the indicator needs to be minimized or maximized.
Also add machinery to only register indicators once so that you can still pass in strings for convenience.

example.ipynb

@@ -2,15 +2,15 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 5,
    "id": "7a828d09-1ddf-43c9-b4d3-4f5d6f8e7fbd",
    "metadata": {},
    "outputs": [],
    "source": [
     "import pandas as pd\n",
     "\n",
     "from pathlib import Path\n",
-    "from cocoviz import ProblemDescription, Result, ResultSet, rtpplot\n",
+    "from cocoviz import ProblemDescription, Result, ResultSet, Indicator, rtpplot\n",
     "\n",
     "DATA_DIR = Path(\"data/\")"
    ]
@@ -25,7 +25,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 6,
    "id": "4e2ec828-3454-4d30-aa8e-5009af42e948",
    "metadata": {},
    "outputs": [],
@@ -55,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 8,
    "id": "17f1ebd9-c24f-4459-9407-d564db0582ef",
    "metadata": {},
    "outputs": [
@@ -82,12 +82,52 @@
    ],
    "source": [
     "number_of_targets = 101\n",
-    "indicator = \"Hypervolume\"\n",
+    "indicator = Indicator(\"Hypervolume\", larger_is_better=True)\n",
     "\n",
     "for problem, result_subset in results.by_number_of_variables():\n",
-    "    ax = rtpplot(result_subset, \"Hypervolume\", number_of_targets=number_of_targets)\n",
+    "    ax = rtpplot(result_subset, indicator, number_of_targets=number_of_targets)\n",
     "    ax.set_title(problem)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7a57f0bf-56dd-4695-9b99-00b769917913",
+   "metadata": {},
+   "source": [
+    "Aggregating over all results will throw an exception."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "d2f34dda-f670-42df-a61d-29874b68cc63",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "BadResultSetException",
+     "evalue": "Cannot derive runtime profile for problems with different number of variables.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mBadResultSetException\u001b[0m                     Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[9], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mrtpplot\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresults\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindicator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumber_of_targets\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnumber_of_targets\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32m~\\work\\cocoviz\\src\\cocoviz\\rtp.py:117\u001b[0m, in \u001b[0;36mrtpplot\u001b[1;34m(results, indicator, number_of_targets, targets, ax)\u001b[0m\n\u001b[0;32m     83\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrtpplot\u001b[39m(\n\u001b[0;32m     84\u001b[0m     results: ResultSet,\n\u001b[0;32m     85\u001b[0m     indicator: \u001b[38;5;28mstr\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     88\u001b[0m     ax\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m     89\u001b[0m ):\n\u001b[0;32m     90\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Plot runtime profiles of the `results`.\u001b[39;00m\n\u001b[0;32m     91\u001b[0m \n\u001b[0;32m     92\u001b[0m \u001b[38;5;124;03m    Parameters\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    115\u001b[0m \u001b[38;5;124;03m        Otherwise a new figure is created and the corresponding Axes object is returned.\u001b[39;00m\n\u001b[0;32m    116\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 117\u001b[0m     profiles \u001b[38;5;241m=\u001b[39m \u001b[43mruntime_profiles\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresults\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindicator\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnumber_of_targets\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnumber_of_targets\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtargets\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtargets\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    118\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m ax \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    119\u001b[0m         fig, ax \u001b[38;5;241m=\u001b[39m plt\u001b[38;5;241m.\u001b[39msubplots()\n",
+      "File \u001b[1;32m~\\work\\cocoviz\\src\\cocoviz\\rtp.py:51\u001b[0m, in \u001b[0;36mruntime_profiles\u001b[1;34m(results, indicator, maximize_indicator, number_of_targets, targets)\u001b[0m\n\u001b[0;32m     48\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m UnknownIndicatorException(indicator)\n\u001b[0;32m     50\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(results\u001b[38;5;241m.\u001b[39mnumber_of_variables) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m---> 51\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m BadResultSetException(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot derive runtime profile for problems with different number of variables.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m     53\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(results\u001b[38;5;241m.\u001b[39mnumber_of_objectives) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m     54\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m BadResultSetException(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot derive runtime profile for problems with different number of objectives.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "\u001b[1;31mBadResultSetException\u001b[0m: Cannot derive runtime profile for problems with different number of variables."
+     ]
+    }
+   ],
+   "source": [
+    "rtpplot(results, indicator, number_of_targets=number_of_targets)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3cd32171-d90f-4e9c-b3e2-20c3c2660b95",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

src/cocoviz/__init__.py

@@ -1,5 +1,6 @@
 from ._version import __version__  # noqa: F401
 from .rtp import rtpplot, runtime_profiles
 from .result import ProblemDescription, Result, ResultSet
+from .indicator import Indicator
 
-__all__ = ["ProblemDescription", "Result", "ResultSet", "rtpplot", "runtime_profiles"]
+__all__ = ["ProblemDescription", "Result", "ResultSet", "Indicator", "rtpplot", "runtime_profiles"]

src/cocoviz/exceptions.py

@@ -14,3 +14,32 @@ class BadResultSetException(Exception):
     """Raised when a runtime profile doesn't make sense for a result set"""    
 
     pass
+
+
+class UnknownIndicatorException(Exception):
+    """Raised when an indicator is passed as a string and hasn't been registered previously"""
+    def __init__(self, name: str):
+        from .indicator import KNOWN_INDICATORS
+        super()
+        self.add_note(f"""You passed the string "{name}" as an indicator that was not previously registered.
+                      
+To use a quality indicator, cocoviz needs to know if the indicator needs to be 
+minimized or maximized. You can either pass in an instance of the Indicator 
+class or register the indicator once with the `register()` function contained 
+in `cocoviz.indicator`.
+
+Currently registered indicators: "{'", "'.join(KNOWN_INDICATORS.keys())}"
+                      
+Example
+-------
+
+>>> from cocoviz import Indicator
+>>> ind = Indicator("hypervolume", larger_is_better=True)
+>>> runtime_profiles(results, indicator=ind)
+
+or
+
+>>> import cocoviz.indicator as ci
+>>> ci.register(ci.Indicator("hypervolume", larger_is_better=True))
+>>> runtime_profiles(results, "hypervolume")                                            
+""")    

src/cocoviz/indicator.py

@@ -0,0 +1,68 @@
+"""Functions and data structures for dealing with performance indicators"""
+
+import dataclasses
+
+from typing import Union
+
+
+KNOWN_INDICATORS = dict()
+
+
+@dataclasses.dataclass(eq=True, frozen=True)
+class Indicator:
+    """Description of a performance indicator
+
+    Attributes
+    ----------
+    name : str
+        Name of the quality indicator. Must match the column name
+        in the Results object.        
+    larger_is_better : bool, default = True
+        True if larger values of the indicator are better, False 
+        otherwise.    
+    """
+    name: str
+    larger_is_better: bool = True
+
+
+def register(ind: Indicator):
+    """Register a new performance indicator
+
+    cocoviz has a global list of know performance indicators with their
+    associated metadata. Using this function, you can add additional indicators
+    so that you do not have to specify their properties every time.
+
+    Parameters
+    ----------
+    ind : Indicator
+        Indicator to add to list of known indicators
+    """
+    KNOWN_INDICATORS[ind.name] = ind
+
+
+def deregister(ind: Union[Indicator, str]):
+    """_summary_
+
+    Parameters
+    ----------
+    ind : Indicator or str
+        Indicator to remove from list of known indicators
+
+    Raises
+    ------
+    NotImplementedError
+        when `ind` is neither a string nor an instance of Indicator
+    """
+    if isinstance(ind, str):
+        del KNOWN_INDICATORS[ind]
+    elif isinstance(ind, Indicator):    
+        del KNOWN_INDICATORS[ind.name]
+    else:
+        raise NotImplementedError()
+
+
+## Register some common and not so common quality indicators
+register(Indicator("hypervolume", larger_is_better=True))
+register(Indicator("hv", larger_is_better=True))
+register(Indicator("r2", larger_is_better=True))
+register(Indicator("time", larger_is_better=False))

src/cocoviz/rtp.py

@@ -4,14 +4,17 @@
 import polars as pl
 import scipy.stats as stats
 
+from typing import Union
+
 from .targets import linear_targets
 from .result import ResultSet
-from .exceptions import BadResultSetException
+from .exceptions import BadResultSetException, UnknownIndicatorException
+from .indicator import Indicator
 
 
 def runtime_profiles(
     results: ResultSet,
-    indicator: str,
+    indicator: Union[Indicator, str],
     maximize_indicator: bool = True,
     number_of_targets: int = 101,
     targets: dict = None,
@@ -22,8 +25,8 @@ def runtime_profiles(
     ----------
     results : ResultSet
         Collection of results of running any number of algorithms on any number of problems or problem instances.
-    indicator : str
-        Name of indicator to analyse.
+    indicator : Indicator or str
+        Performance indicator to analyse.
     maximize_indicator : bool, optional
         Should the indicator be maximized or minimized?
     number_of_targets : int, optional
@@ -37,7 +40,13 @@ def runtime_profiles(
     dict
         Quantiles and probabilities for each algorithm in `results`.
     """
-
+    if not isinstance(indicator, Indicator):
+        try:
+            from .indicator import KNOWN_INDICATORS
+            indicator = KNOWN_INDICATORS[indicator]
+        except KeyError:
+            raise UnknownIndicatorException(indicator)
+
     if len(results.number_of_variables) > 1:
         raise BadResultSetException("Cannot derive runtime profile for problems with different number of variables.")
 
@@ -46,12 +55,12 @@ def runtime_profiles(
 
     # If no targets are given, calculate `number_of_targets` linearly spaced targets
     if not targets:
-        targets = linear_targets(results, indicator, number_of_targets)
+        targets = linear_targets(results, indicator.name, number_of_targets)
 
     # Get (approximate) runtime to reach each target of indicator
     indicator_results = ResultSet()
     for r in results:
-        indicator_results.append(r.at_indicator(indicator, targets[r.problem]))
+        indicator_results.append(r.at_indicator(indicator.name, targets[r.problem]))
 
     res = {}
     for algo, algo_results in indicator_results.by_algorithm():
@@ -84,8 +93,10 @@ def rtpplot(
     ----------
     results : ResultSet
         Collection of results of running any number of algorithms on any number of problems or problem instances.
-    indicator : str
-        Name of indicator to analyse.
+    indicator : Indicator or str
+        Performance indicator to analyse. Name must match the name used in the results.
+        If a string is passed in, the indicator must have been registered previously by a call to
+        `cocoviz.indicator.register`.
     number_of_targets : int, optional
         Number of target values to generate for each problem it `targets` is missing.
     targets : dict, optional