Add max_broken in exp configuration

The experiment was using `worker.max_broken` but max_broken for the
experiment is also configurable and should be handled differently.

Author: bouthilx

src/orion/client/__init__.py

@@ -24,7 +24,7 @@
 # pylint: disable=too-many-arguments
 def create_experiment(
         name, version=None, space=None, algorithms=None,
-        strategy=None, max_trials=None, storage=None, branching=None,
+        strategy=None, max_trials=None, max_broken=None, storage=None, branching=None,
         max_idle_time=None, heartbeat=None, working_dir=None, debug=False):
     """Create an experiment
 
@@ -53,12 +53,12 @@ def create_experiment(
     2.2) Some other arguments than the name are given.
 
     The configuration will be fetched from database and given arguments will override them.
-    ``max_trials`` may be overwritten in DB, but any other changes will lead to a branching. Instead
-    of creating the experiment ``(name, version)``, it will create a new experiment
-    ``(name, version+1)`` which will have the same configuration than ``(name, version)`` except for
-    the differing arguments given by user. This new experiment will have access to trials of
-    ``(name, version)``, adapted according to the differences between ``version`` and ``version+1``.
-    A previous version can be accessed by specifying the ``version`` argument.
+    ``max_trials`` and ``max_broken`` may be overwritten in DB, but any other changes will lead to a
+    branching. Instead of creating the experiment ``(name, version)``, it will create a new
+    experiment ``(name, version+1)`` which will have the same configuration than ``(name, version)``
+    except for the differing arguments given by user. This new experiment will have access to trials
+    of ``(name, version)``, adapted according to the differences between ``version`` and
+    ``version+1``.  A previous version can be accessed by specifying the ``version`` argument.
 
     Causes of experiment branching are:
 
@@ -91,6 +91,8 @@ def create_experiment(
         Parallel strategy to use to parallelize the algorithm.
     max_trials: int, optional
         Maximum number or trials before the experiment is considered done.
+    max_broken: int, optional
+        Number of broken trials for the experiment to be considered broken.
     storage: dict, optional
         Configuration of the storage backend.
     working_dir: str, optional
@@ -163,16 +165,16 @@ def create_experiment(
     try:
         experiment = experiment_builder.build(
             name, version=version, space=space, algorithms=algorithms,
-            strategy=strategy, max_trials=max_trials, branching=branching,
+            strategy=strategy, max_trials=max_trials, max_broken=max_broken, branching=branching,
             working_dir=working_dir)
     except RaceCondition:
         # Try again, but if it fails again, raise. Race conditions due to version increment should
         # only occur once in a short window of time unless code version is changing at a crazy pace.
         try:
             experiment = experiment_builder.build(
                 name, version=version, space=space, algorithms=algorithms,
-                strategy=strategy, max_trials=max_trials, branching=branching,
-                working_dir=working_dir)
+                strategy=strategy, max_trials=max_trials, max_broken=max_broken,
+                branching=branching, working_dir=working_dir)
         except RaceCondition as e:
             raise RaceCondition(
                 "There was a race condition during branching and new version cannot be infered "
@@ -214,7 +216,7 @@ def get_experiment(name, version=None, storage=None):
     return experiment_builder.build_view(name, version)
 
 
-def workon(function, space, name='loop', algorithms=None, max_trials=None):
+def workon(function, space, name='loop', algorithms=None, max_trials=None, max_broken=None):
     """Optimize a function over a given search space
 
     This will create a new experiment with an in-memory storage and optimize the given function
@@ -241,6 +243,8 @@ def workon(function, space, name='loop', algorithms=None, max_trials=None):
         Algorithm used for optimization.
     max_trials: int, optional
         Maximum number or trials before the experiment is considered done.
+    max_broken: int, optional
+        Number of broken trials for the experiment to be considered broken.
 
     Raises
     ------
@@ -256,7 +260,7 @@ def workon(function, space, name='loop', algorithms=None, max_trials=None):
 
         experiment = experiment_builder.build(
             name, version=1, space=space, algorithms=algorithms,
-            strategy='NoParallelStrategy', max_trials=max_trials)
+            strategy='NoParallelStrategy', max_trials=max_trials, max_broken=max_broken)
 
         producer = Producer(experiment)
 

src/orion/client/experiment.py

@@ -96,6 +96,11 @@ def max_trials(self):
         """Max-trials to execute before stopping the experiment."""
         return self._experiment.max_trials
 
+    @property
+    def max_broken(self):
+        """Minimum number of broken trials before the experiment is considered broken."""
+        return self._experiment.max_broken
+
     @property
     def metadata(self):
         """Metadata of the experiment."""

src/orion/core/io/experiment_builder.py

@@ -138,7 +138,9 @@ def build(name, version=None, branching=None, **config):
     strategy: str or dict, optional
         Parallel strategy to use to parallelize the algorithm.
     max_trials: int, optional
-        Maximum number or trials before the experiment is considered done.
+        Maximum number of trials before the experiment is considered done.
+    max_broken: int, optional
+        Number of broken trials for the experiment to be considered broken.
     storage: dict, optional
         Configuration of the storage backend.
     branching: dict, optional
@@ -277,6 +279,8 @@ def create_experiment(name, version, space, **kwargs):
         Parallel strategy to use to parallelize the algorithm.
     max_trials: int, optional
         Maximum number or trials before the experiment is considered done.
+    max_broken: int, optional
+        Number of broken trials for the experiment to be considered broken.
     storage: dict, optional
         Configuration of the storage backend.
 
@@ -288,6 +292,7 @@ def create_experiment(name, version, space, **kwargs):
         experiment.pool_size = orion.core.config.experiment.get(
             'pool_size', deprecated='ignore')
     experiment.max_trials = kwargs.get('max_trials', orion.core.config.experiment.max_trials)
+    experiment.max_broken = kwargs.get('max_broken', orion.core.config.experiment.max_broken)
     experiment.space = _instantiate_space(space)
     experiment.algorithms = _instantiate_algo(experiment.space, kwargs.get('algorithms'))
     experiment.producer = kwargs.get('producer', {})
@@ -324,6 +329,7 @@ def fetch_config_from_db(name, version=None):
                  "version %s.", name, config['version'])
 
     backward.populate_space(config, force_update=False)
+    backward.update_max_broken(config)
 
     return config
 

src/orion/core/utils/backward.py

@@ -34,6 +34,12 @@ def populate_priors(metadata):
     metadata["priors"] = dict(parser.priors)
 
 
+def update_max_broken(config):
+    """Set default max_broken if None (in v <= v0.1.9)"""
+    if not config.get('max_broken', None):
+        config['max_broken'] = orion.core.config.experiment.max_broken
+
+
 def populate_space(config, force_update=True):
     """Add the space definition at the root of config."""
     if 'space' in config and not force_update:

src/orion/core/utils/format_terminal.py

@@ -258,6 +258,7 @@ def format_commandline(experiment):
 {title}
 pool size: {experiment.pool_size}
 max trials: {experiment.max_trials}
+max broken: {experiment.max_broken}
 working dir: {experiment.working_dir}
 """
 

src/orion/core/worker/experiment.py

@@ -13,7 +13,6 @@
 import datetime
 import logging
 
-import orion.core
 from orion.core.evc.adapters import BaseAdapter
 from orion.core.evc.experiment import ExperimentNode
 from orion.storage.base import FailedUpdate, get_storage, ReadOnlyStorageProtocol
@@ -48,6 +47,11 @@ class Experiment:
        This attribute can be updated if the rest of the experiment configuration
        is the same. In that case, if trying to set to an already set experiment,
        it will overwrite the previous one.
+    max_broken: int
+       How many trials must be broken, before considering this `Experiment` broken.
+       This attribute can be updated if the rest of the experiment configuration
+       is the same. In that case, if trying to set to an already set experiment,
+       it will overwrite the previous one.
     space: Space
        Object representing the optimization space.
     algorithms : `PrimaryAlgo` object.
@@ -77,10 +81,10 @@ class Experiment:
 
     """
 
-    __slots__ = ('name', 'refers', 'metadata', 'pool_size', 'max_trials', 'version',
+    __slots__ = ('name', 'refers', 'metadata', 'pool_size', 'max_trials', 'max_broken', 'version',
                  'space', 'algorithms', 'producer', 'working_dir', '_id',
                  '_node', '_storage')
-    non_branching_attrs = ('pool_size', 'max_trials')
+    non_branching_attrs = ('pool_size', 'max_trials', 'max_broken')
 
     def __init__(self, name, version=None):
         self._id = None
@@ -91,6 +95,7 @@ def __init__(self, name, version=None):
         self.metadata = {}
         self.pool_size = None
         self.max_trials = None
+        self.max_broken = None
         self.space = None
         self.algorithms = None
         self.working_dir = None
@@ -316,7 +321,7 @@ def is_broken(self):
 
         """
         num_broken_trials = self._storage.count_broken_trials(self)
-        return num_broken_trials >= orion.core.config.worker.max_broken
+        return num_broken_trials >= self.max_broken
 
     @property
     def configuration(self):
@@ -409,7 +414,7 @@ class ExperimentView(object):
 
     #                     Attributes
     valid_attributes = (["_id", "name", "refers", "metadata", "pool_size", "max_trials",
-                         "version", "space", "working_dir"] +
+                         "max_broken", "version", "space", "working_dir"] +
                         # Properties
                         ["id", "node", "is_done", "is_broken", "algorithms", "stats",
                          "configuration"] +

src/orion/serving/responses.py

@@ -73,6 +73,7 @@ def build_experiment_response(experiment: Experiment,
         "orionVersion": experiment.metadata['orion_version'],
         "config": {
             "maxTrials": experiment.max_trials,
+            "maxBroken": experiment.max_broken,
             "poolSize": experiment.pool_size,
             "algorithm": algorithm,
             "space": experiment.configuration['space']

tests/functional/demo/orion_config.yaml

@@ -2,6 +2,7 @@ name: voila_voici
 
 pool_size: 1
 max_trials: 100
+max_broken: 5
 
 algorithms:
   gradient_descent:

tests/functional/demo/orion_config_other.yaml

@@ -2,6 +2,7 @@ name: voila_voici
 
 pool_size: 1
 max_trials: 100
+max_broken: 5
 
 algorithms:
   gradient_descent:

tests/functional/demo/orion_config_random.yaml

@@ -2,6 +2,7 @@ name: demo_random_search
 
 pool_size: 2
 max_trials: 400
+max_broken: 5
 
 algorithms: random
 

tests/functional/demo/test_demo.py

@@ -39,6 +39,7 @@ def test_demo_with_default_algo_cli_config_only(database, monkeypatch):
     assert exp['name'] == 'default_algo'
     assert exp['pool_size'] == 1
     assert exp['max_trials'] == 30
+    assert exp['max_broken'] == 3
     assert exp['algorithms'] == {'random': {'seed': None}}
     assert 'user' in exp['metadata']
     assert 'datetime' in exp['metadata']
@@ -74,6 +75,7 @@ def test_demo(database, monkeypatch):
     assert exp['name'] == 'voila_voici'
     assert exp['pool_size'] == 1
     assert exp['max_trials'] == 100
+    assert exp['max_broken'] == 5
     assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1,
                                                       'dx_tolerance': 1e-7}}
     assert 'user' in exp['metadata']
@@ -117,6 +119,7 @@ def test_demo_with_script_config(database, monkeypatch):
     assert exp['name'] == 'voila_voici'
     assert exp['pool_size'] == 1
     assert exp['max_trials'] == 100
+    assert exp['max_broken'] == 5
     assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1,
                                                       'dx_tolerance': 1e-7}}
     assert 'user' in exp['metadata']
@@ -162,6 +165,7 @@ def test_demo_with_python_and_script(database, monkeypatch):
     assert exp['name'] == 'voila_voici'
     assert exp['pool_size'] == 1
     assert exp['max_trials'] == 100
+    assert exp['max_broken'] == 5
     assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1,
                                                       'dx_tolerance': 1e-7}}
     assert 'user' in exp['metadata']
@@ -227,6 +231,7 @@ def test_demo_two_workers(database, monkeypatch):
     assert exp['name'] == 'two_workers_demo'
     assert exp['pool_size'] == 2
     assert exp['max_trials'] == 100
+    assert exp['max_broken'] == 5
     assert exp['algorithms'] == {'random': {'seed': None}}
     assert 'user' in exp['metadata']
     assert 'datetime' in exp['metadata']
@@ -257,6 +262,7 @@ def test_workon():
         }
     config['pool_size'] = 1
     config['max_trials'] = 100
+    config['exp_max_broken'] = 5
     config['user_args'] = [
         os.path.abspath(os.path.join(os.path.dirname(__file__), "black_box.py")),
         "-x~uniform(-50, 50, precision=None)"]
@@ -275,6 +281,7 @@ def test_workon():
         assert exp['name'] == name
         assert exp['pool_size'] == 1
         assert exp['max_trials'] == 100
+        assert exp['max_broken'] == 5
         assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1,
                                                           'dx_tolerance': 1e-7}}
         assert 'user' in exp['metadata']
@@ -473,7 +480,6 @@ def test_run_with_parallel_strategy(database, monkeypatch, strategy):
     assert len(exp) == 1
     exp = exp[0]
     assert exp['producer']['strategy'] == strategy
-    print(exp['max_trials'])
     assert '_id' in exp
     exp_id = exp['_id']
     trials = list(database.trials.find({'experiment': exp_id}))

tests/functional/serving/test_experiments_resource.py

@@ -22,6 +22,7 @@
     version=1,
     pool_size=1,
     max_trials=10,
+    max_broken=7,
     working_dir='',
     algorithms={'random': {'seed': 1}},
     producer={'strategy': 'NoParallelStrategy'},
@@ -199,6 +200,7 @@ def _assert_config(config):
     """Asserts properties of the ``config`` dictionary"""
     assert config['poolSize'] == 1
     assert config['maxTrials'] == 10
+    assert config['maxBroken'] == 7
 
     algorithm = config['algorithm']
     assert algorithm['name'] == 'random'

tests/unittests/client/test_client.py

@@ -37,6 +37,7 @@
     version=1,
     pool_size=1,
     max_trials=10,
+    max_broken=5,
     working_dir='',
     algorithms={'random': {'seed': 1}},
     producer={'strategy': 'NoParallelStrategy'},
@@ -195,6 +196,7 @@ def test_create_experiment_new_default(self):
             assert experiment.space.configuration == space
 
             assert experiment.max_trials == orion.core.config.experiment.max_trials
+            assert experiment.max_broken == orion.core.config.experiment.max_broken
             assert experiment.working_dir == orion.core.config.experiment.working_dir
             assert experiment.algorithms.configuration == {'random': {'seed': None}}
             assert experiment.configuration['producer'] == {'strategy': 'MaxParallelStrategy'}
@@ -208,6 +210,7 @@ def test_create_experiment_new_full_config(self, user_config):
 
             assert exp_config['space'] == config['space']
             assert exp_config['max_trials'] == config['max_trials']
+            assert exp_config['max_broken'] == config['max_broken']
             assert exp_config['working_dir'] == config['working_dir']
             assert exp_config['algorithms'] == config['algorithms']
             assert exp_config['producer'] == config['producer']
@@ -223,6 +226,7 @@ def test_create_experiment_hit_no_branch(self, user_config):
             assert experiment.version == 1
             assert exp_config['space'] == config['space']
             assert exp_config['max_trials'] == config['max_trials']
+            assert exp_config['max_broken'] == config['max_broken']
             assert exp_config['working_dir'] == config['working_dir']
             assert exp_config['algorithms'] == config['algorithms']
             assert exp_config['producer'] == config['producer']
@@ -237,6 +241,7 @@ def test_create_experiment_hit_no_config(self):
             assert experiment.space.configuration == config['space']
             assert experiment.algorithms.configuration == config['algorithms']
             assert experiment.max_trials == config['max_trials']
+            assert experiment.max_broken == config['max_broken']
             assert experiment.working_dir == config['working_dir']
             assert experiment.producer['strategy'].configuration == config['producer']['strategy']
 
@@ -250,6 +255,7 @@ def test_create_experiment_hit_branch(self):
 
             assert experiment.algorithms.configuration == config['algorithms']
             assert experiment.max_trials == config['max_trials']
+            assert experiment.max_broken == config['max_broken']
             assert experiment.working_dir == config['working_dir']
             assert experiment.producer['strategy'].configuration == config['producer']['strategy']
 

tests/unittests/client/test_experiment_client.py

@@ -28,6 +28,7 @@
     version=1,
     pool_size=1,
     max_trials=10,
+    max_broken=5,
     working_dir='',
     algorithms={'random': {'seed': 1}},
     producer={'strategy': 'NoParallelStrategy'},