Merge pull request #266 from bouthilx/feature/algo_opt_out

Add to algo the ability to opt out

src/orion/algo/base.py

@@ -143,10 +143,22 @@ def set_state(self, state_dict):
     def suggest(self, num=1):
         """Suggest a `num` of new sets of parameters.
 
-        :param num: how many sets to be suggested.
+        Parameters
+        ----------
+        num: int, optional
+            Number of points to suggest. Defaults to 1.
+
+        Returns
+        -------
+        list of points or None
+            A list of lists representing points suggested by the algorithm. The algorithm may opt
+            out if it cannot make a good suggestion at the moment (it may be waiting for other
+            trials to complete), in which case it will return None.
+
+        Notes
+        -----
+        New parameters must be compliant with the problem's domain `orion.algo.space.Space`.
 
-        .. note:: New parameters must be compliant with the problem's domain
-           `orion.algo.space.Space`.
         """
         pass
 

src/orion/core/__init__.py

@@ -35,7 +35,7 @@
 __author__ = u'Epistímio'
 __author_short__ = u'Epistímio'
 __author_email__ = '[email protected]'
-__copyright__ = u'2017-2018, Epistímio'
+__copyright__ = u'2017-2019, Epistímio'
 __url__ = 'https://github.com/epistimio/orion'
 
 DIRS = AppDirs(__name__, __author_short__)
@@ -89,6 +89,8 @@ def define_worker_config(config):
         'heartbeat', option_type=int, default=120)
     worker_config.add_option(
         'max_broken', option_type=int, default=3)
+    worker_config.add_option(
+        'max_idle_time', option_type=int, default=60)
 
     config.worker = worker_config
 

src/orion/core/worker/primary_algo.py

@@ -67,12 +67,17 @@ def suggest(self, num=1):
            `orion.algo.space.Space`.
         """
         points = self.algorithm.suggest(num)
+
+        if points is None:
+            return None
+
         for point in points:
             if point not in self.transformed_space:
                 raise ValueError("""
 Point is not contained in space:
 Point: {}
 Space: {}""".format(point, self.transformed_space))
+
         return [self.transformed_space.reverse(point) for point in points]
 
     def observe(self, points, results):

src/orion/core/worker/producer.py

@@ -10,7 +10,10 @@
 """
 import copy
 import logging
+import random
+import time
 
+import orion.core
 from orion.core.io.database import DuplicateKeyError
 from orion.core.utils import format_trials
 from orion.core.worker.trials_history import TrialsHistory
@@ -27,7 +30,7 @@ class Producer(object):
 
     """
 
-    def __init__(self, experiment, max_attempts=100):
+    def __init__(self, experiment, max_idle_time=None):
         """Initialize a producer.
 
         :param experiment: Manager of this experiment, provides convenient
@@ -40,7 +43,9 @@ def __init__(self, experiment, max_attempts=100):
             raise RuntimeError("Experiment object provided to Producer has not yet completed"
                                " initialization.")
         self.algorithm = experiment.algorithms
-        self.max_attempts = max_attempts
+        if max_idle_time is None:
+            max_idle_time = orion.core.config.worker.max_idle_time
+        self.max_idle_time = max_idle_time
         self.strategy = experiment.producer['strategy']
         self.naive_algorithm = None
         # TODO: Move trials_history into PrimaryAlgo during the refactoring of Algorithm with
@@ -53,18 +58,34 @@ def pool_size(self):
         """Pool-size of the experiment"""
         return self.experiment.pool_size
 
+    def backoff(self):
+        """Wait some time and update algorithm."""
+        waiting_time = min(0, random.gauss(1, 0.2))
+        log.info('Waiting %d seconds', waiting_time)
+        time.sleep(waiting_time)
+        log.info('Updating algorithm.')
+        self.update()
+
     def produce(self):
         """Create and register new trials."""
         sampled_points = 0
-        n_attempts = 0
 
-        while sampled_points < self.pool_size and n_attempts < self.max_attempts:
-            n_attempts += 1
+        start = time.time()
+        while sampled_points < self.pool_size:
+            if time.time() - start > self.max_idle_time:
+                raise RuntimeError(
+                    "Algorithm could not sample new points in less than {} seconds".format(
+                        self.max_idle_time))
+
             log.debug("### Algorithm suggests new points.")
 
             new_points = self.naive_algorithm.suggest(self.pool_size)
             # Sync state of original algo so that state continues evolving.
             self.algorithm.set_state(self.naive_algorithm.state_dict)
+            if new_points is None:
+                log.info("### Algo opted out.")
+                self.backoff()
+                continue
 
             for new_point in new_points:
                 log.debug("#### Convert point to `Trial` object.")
@@ -75,19 +96,10 @@ def produce(self):
                     self.experiment.register_trial(new_trial)
                     sampled_points += 1
                 except DuplicateKeyError:
-                    log.debug("#### Duplicate sample. Updating algo to produce new ones.")
-                    self.update()
+                    log.debug("#### Duplicate sample.")
+                    self.backoff()
                     break
 
-        if n_attempts >= self.max_attempts:
-            raise RuntimeError("Looks like the algorithm keeps suggesting trial configurations"
-                               "that already exist in the database. Could be that you reached "
-                               "a point of convergence and the algorithm cannot find anything "
-                               "better. Or... something is broken. Try increasing `max_attempts` "
-                               "or please report this error on "
-                               "https://github.com/epistimio/orion/issues if something looks "
-                               "wrong.")
-
     def update(self):
         """Pull all trials to update model with completed ones and naive model with non completed
         ones.

tests/unittests/core/test_producer.py

@@ -3,6 +3,7 @@
 """Collection of tests for :mod:`orion.core.worker.producer`."""
 import copy
 import datetime
+import time
 
 import pytest
 
@@ -534,18 +535,46 @@ def test_duplicate_within_pool_and_db(producer, database, random_dt):
         ]
 
 
-def test_exceed_max_attempts(producer, database, random_dt):
+def test_exceed_max_idle_time_because_of_duplicates(producer, database, random_dt):
     """Test that RuntimeError is raised when algo keep suggesting the same points"""
-    producer.max_attempts = 10  # to limit run-time, default would work as well.
+    timeout = 3
+    producer.max_idle_time = timeout  # to limit run-time, default would work as well.
     producer.experiment.algorithms.algorithm.possible_values = [('rnn', 'rnn')]
 
     assert producer.experiment.pool_size == 1
 
     producer.update()
 
+    start = time.time()
     with pytest.raises(RuntimeError) as exc_info:
         producer.produce()
-    assert "Looks like the algorithm keeps suggesting" in str(exc_info.value)
+    assert timeout <= time.time() - start < timeout + 1
+
+    assert "Algorithm could not sample new points" in str(exc_info.value)
+
+
+def test_exceed_max_idle_time_because_of_optout(producer, database, random_dt, monkeypatch):
+    """Test that RuntimeError is raised when algo keeps opting out"""
+    timeout = 3
+    producer.max_idle_time = timeout  # to limit run-time, default would work as well.
+
+    def opt_out(self, num=1):
+        """Return None to always opt out"""
+        self._suggested = None
+        return None
+
+    monkeypatch.setattr(producer.experiment.algorithms.algorithm.__class__, 'suggest', opt_out)
+
+    assert producer.experiment.pool_size == 1
+
+    producer.update()
+
+    start = time.time()
+    with pytest.raises(RuntimeError) as exc_info:
+        producer.produce()
+    assert timeout <= time.time() - start < timeout + 1
+
+    assert "Algorithm could not sample new points" in str(exc_info.value)
 
 
 def test_original_seeding(producer, database):