Deterministic algorithm benchmark

docs/src/user/benchmark.rst

@@ -15,7 +15,7 @@ for how to create and :doc:`/code/benchmark` for how to use benchmark.
   from orion.benchmark.task import RosenBrock, EggHolder, CarromTable
 
   benchmark = get_or_create_benchmark(name='benchmark',
-          algorithms=['random', 'tpe'],
+          algorithms=[{'algorithm': 'random'}, {'algorithm': 'tpe'}],
           targets=[
               {
                   'assess': [AverageResult(2), AverageRank(2)],

src/orion/benchmark/__init__.py

@@ -21,7 +21,18 @@ class Benchmark:
     name: str
         Name of the benchmark
     algorithms: list, optional
-        Algorithms used for benchmark, each algorithm can be a string or dict.
+        Algorithms used for benchmark, with supported formats:
+
+        - A `str` of the algorithm name
+        - A `dict`, with only one key and one value, where key is the algorithm name and value is a dict for the algorithm config.
+        - A `dict`, with two keys.
+
+            algorithm: str or dict
+                Algorithm name in string or a dict with algorithm configure.
+            deterministic: bool, optional
+                True if it is a deterministic algorithm, then for each assessment, only one experiment
+                will be run for this algorithm.
+
     targets: list, optional
         Targets for the benchmark, each target will be a dict with two keys.
 
@@ -195,17 +206,66 @@ class Study:
 
     Parameters
     ----------
-    benchmark: `Benchmark` instance
+    benchmark: A Benchmark instance
     algorithms: list
-        Algorithms used for benchmark, each algorithm can be a string or dict.
+        Algorithms used for benchmark, each algorithm can be a string or dict, with same format as `Benchmark` algorithms.
     assessment: list
         `Assessment` instance
     task: list
         `Task` instance
     """
 
+    class _StudyAlgorithm:
+        """
+        Represent user input json format algorithm as a Study algorithm object for easy to use.
+        Parameters
+        ----------
+        algorithm: one algorithm in the `Study` algorithms list.
+        """
+
+        def __init__(self, algorithm):
+            parameters = None
+            deterministic = False
+
+            if isinstance(algorithm, dict):
+                if len(algorithm) > 1 or algorithm.get("algorithm"):
+                    print(algorithm)
+                    deterministic = algorithm.get("deterministic", False)
+                    experiment_algorithm = algorithm["algorithm"]
+
+                    if isinstance(experiment_algorithm, dict):
+                        name, parameters = list(experiment_algorithm.items())[0]
+                    else:
+                        name = experiment_algorithm
+                else:
+                    print(list(algorithm.items())[0])
+                    name, parameters = list(algorithm.items())[0]
+                    print(name, parameters)
+            else:
+                print(algorithm)
+                name = algorithm
+
+            self.algo_name = name
+            self.parameters = parameters
+            self.deterministic = deterministic
+
+        @property
+        def name(self):
+            return self.algo_name
+
+        @property
+        def experiment_algorithm(self):
+            if self.parameters:
+                return {self.algo_name: self.parameters}
+            else:
+                return self.algo_name
+
+        @property
+        def is_deterministic(self):
+            return self.deterministic
+
     def __init__(self, benchmark, algorithms, assessment, task):
-        self.algorithms = algorithms
+        self.algorithms = self._build_benchmark_algorithms(algorithms)  # algorithms
         self.assessment = assessment
         self.task = task
         self.benchmark = benchmark
@@ -214,6 +274,13 @@ def __init__(self, benchmark, algorithms, assessment, task):
         self.task_name = type(self.task).__name__
         self.experiments_info = []
 
+    def _build_benchmark_algorithms(self, algorithms):
+        benchmark_algorithms = list()
+        for algorithm in algorithms:
+            benchmark_algorithm = self._StudyAlgorithm(algorithm)
+            benchmark_algorithms.append(benchmark_algorithm)
+        return benchmark_algorithms
+
     def setup_experiments(self):
         """Setup experiments to run of the study"""
         max_trials = self.task.max_trials
@@ -224,6 +291,13 @@ def setup_experiments(self):
 
             for algo_index, algorithm in enumerate(self.algorithms):
 
+                # deterministic = algorithm.get("deterministic", False)
+                # algorithm = algorithm["algorithm"]
+
+                # Run only 1 experiment for deterministic algorithm
+                if algorithm.is_deterministic and task_index > 0:
+                    continue
+
                 experiment_name = (
                     self.benchmark.name
                     + "_"
@@ -235,10 +309,11 @@ def setup_experiments(self):
                     + "_"
                     + str(algo_index)
                 )
+
                 experiment = create_experiment(
                     experiment_name,
                     space=space,
-                    algorithms=algorithm,
+                    algorithms=algorithm.experiment_algorithm,
                     max_trials=max_trials,
                 )
                 self.experiments_info.append((task_index, experiment))
@@ -294,13 +369,16 @@ def experiments(self):
 
     def __repr__(self):
         """Represent the object as a string."""
-        algorithms_list = list()
+        algorithms_list = [algorithm.name for algorithm in self.algorithms]
+        """
         for algorithm in self.algorithms:
-            if isinstance(algorithm, dict):
-                algorithm_name = list(algorithm.keys())[0]
+            algo = algorithm["algorithm"]
+            if isinstance(algo, dict):
+                algorithm_name = list(algo.keys())[0]
             else:
-                algorithm_name = algorithm
+                algorithm_name = algo
             algorithms_list.append(algorithm_name)
+        """
 
         return "Study(assessment=%s, task=%s, algorithms=[%s])" % (
             self.assess_name,

src/orion/testing/__init__.py

@@ -84,7 +84,7 @@ def generate_benchmark_experiments_trials(
         exp = copy.deepcopy(experiment_config)
         exp["_id"] = i
         exp["name"] = "experiment-name-{}".format(i)
-        exp["algorithms"] = benchmark_algorithms[i % algo_num]
+        exp["algorithms"] = benchmark_algorithms[i % algo_num]["algorithm"]
         exp["max_trials"] = max_trial
         exp["metadata"]["datetime"] = datetime.datetime.utcnow()
         gen_exps.append(exp)

tests/functional/benchmark/test_benchmark_flow.py

@@ -8,7 +8,10 @@
 from orion.benchmark.benchmark_client import get_or_create_benchmark
 from orion.benchmark.task import BaseTask, Branin, CarromTable, EggHolder, RosenBrock
 
-algorithms = [{"random": {"seed": 1}}, {"tpe": {"seed": 1}}]
+algorithms = [
+    {"algorithm": {"random": {"seed": 1}}},
+    {"algorithm": {"tpe": {"seed": 1}}},
+]
 
 
 class BirdLike(BaseTask):

tests/unittests/benchmark/conftest.py

@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 """Common fixtures and utils for benchmark unit tests."""
 
-import datetime
-
 import pytest
 
 from orion.benchmark.assessment import AverageRank, AverageResult
@@ -26,7 +24,7 @@ def max_trial():
 @pytest.fixture
 def benchmark_algorithms():
     """Return a list of algorithms suitable for Orion experiment"""
-    return [{"random": {"seed": 1}}, {"tpe": {"seed": 1}}]
+    return [{"algorithm": {"random": {"seed": 1}}}, {"algorithm": {"tpe": {"seed": 1}}}]
 
 
 @pytest.fixture
@@ -87,7 +85,7 @@ def experiment_config(max_trial, benchmark_algorithms):
         pool_size=1,
         max_trials=max_trial,
         working_dir="",
-        algorithms=benchmark_algorithms[0],
+        algorithms=benchmark_algorithms[0]["algorithm"],
         producer={"strategy": "NoParallelStrategy"},
     )
     return config

tests/unittests/benchmark/test_assessments.py

@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 """Tests for :mod:`orion.benchmark.assessment`."""
 
-import copy
-
 import plotly
 import pytest
 
@@ -49,7 +47,7 @@ def test_figure_layout(self, study_experiments_config):
             assert_rankings_plot(
                 plot,
                 [
-                    list(algorithm.keys())[0]
+                    list(algorithm["algorithm"].keys())[0]
                     for algorithm in study_experiments_config["algorithms"]
                 ],
                 balanced=study_experiments_config["max_trial"],
@@ -94,7 +92,7 @@ def test_figure_layout(self, study_experiments_config):
             assert_regrets_plot(
                 plot,
                 [
-                    list(algorithm.keys())[0]
+                    list(algorithm["algorithm"].keys())[0]
                     for algorithm in study_experiments_config["algorithms"]
                 ],
                 balanced=study_experiments_config["max_trial"],

tests/unittests/benchmark/test_benchmark.py

@@ -189,6 +189,34 @@ def test_creation(self, study):
             "task=RosenBrock, algorithms=[random,tpe])"
         )
 
+    def test_creation_algorithms(self, benchmark):
+        """Test study creation with all support algorithms input format"""
+
+        algorithms = [
+            {"algorithm": {"gridsearch": {"n_values": 1}}, "deterministic": True},
+            {"algorithm": "tpe"},
+            {"random": {"seed": 1}},
+            "asha",
+        ]
+        study = Study(benchmark, algorithms, AverageResult(2), RosenBrock(25, dim=3))
+        assert study.algorithms[0].name == "gridsearch"
+        assert study.algorithms[0].experiment_algorithm == {
+            "gridsearch": {"n_values": 1}
+        }
+        assert study.algorithms[0].is_deterministic
+
+        assert study.algorithms[1].name == "tpe"
+        assert study.algorithms[1].experiment_algorithm == "tpe"
+        assert not study.algorithms[1].is_deterministic
+
+        assert study.algorithms[2].name == "random"
+        assert study.algorithms[2].experiment_algorithm == {"random": {"seed": 1}}
+        assert not study.algorithms[2].is_deterministic
+
+        assert study.algorithms[3].name == "asha"
+        assert study.algorithms[3].experiment_algorithm == "asha"
+        assert not study.algorithms[3].is_deterministic
+
     def test_setup_experiments(self, study):
         """Test to setup experiments for study"""
         with OrionState():

tests/unittests/benchmark/test_benchmark_client.py

@@ -169,10 +169,28 @@ def test_create_with_invalid_algorithms(self, benchmark_config_py):
         with OrionState():
 
             with pytest.raises(NotImplementedError) as exc:
-                benchmark_config_py["algorithms"] = [{"fake_algorithm": {"seed": 1}}]
+                benchmark_config_py["algorithms"] = [
+                    {"algorithm": {"fake_algorithm": {"seed": 1}}}
+                ]
                 get_or_create_benchmark(**benchmark_config_py)
             assert "Could not find implementation of BaseAlgorithm" in str(exc.value)
 
+    def test_create_with_deterministic_algorithm(self, benchmark_config_py):
+        algorithms = [
+            {"algorithm": {"random": {"seed": 1}}},
+            {"algorithm": {"gridsearch": {"n_values": 50}}, "deterministic": True},
+        ]
+        with OrionState():
+            config = copy.deepcopy(benchmark_config_py)
+            config["algorithms"] = algorithms
+            bm = get_or_create_benchmark(**config)
+
+            for study in bm.studies:
+                for status in study.status():
+                    algo = status["algorithm"]
+                    if algo == "gridsearch":
+                        assert status["experiments"] == 1
+
     def test_create_with_invalid_targets(self, benchmark_config_py):
         """Test creation with invalid Task and Assessment"""
         with OrionState():