Fix ASHA termination condition and branching with fidelity()

docs/src/user/algorithms.rst

@@ -2,6 +2,10 @@
 Setup Algorithms
 ****************
 
+.. contents::
+   :depth: 2
+   :local:
+
 Default algorithm is a random search based on the probability
 distribution given to a search parameter's definition.
 
@@ -27,6 +31,12 @@ yaml file as shown above with ``learning_rate``.
 Included Algorithms
 ===================
 
+.. contents::
+   :depth: 1
+   :local:
+
+.. _random-search:
+
 Random Search
 -------------
 
@@ -86,11 +96,21 @@ Configuration
 
 .. code-block:: yaml
 
-     algorithms:
-        asha:
-           seed: null
-           num_rungs: null
-           num_brackets: 1
+    algorithms:
+       asha:
+          seed: null
+          num_rungs: null
+          num_brackets: 1
+
+    producer:
+      strategy: StubParallelStrategy
+
+
+.. note::
+
+   Notice the additional ``producer.strategy`` in configuration which is not mandatory for other
+   algorithms. See :ref:`StubParallelStrategy` for more information.
+
 
 ``seed``
 
@@ -110,10 +130,11 @@ converging trials that do not lead to best results at convergence (stragglers).
 To overcome this, you can increase the number of brackets, which increases the amount of resources
 required for optimisation but decreases the bias towards stragglers. Default is 1.
 
-
 Algorithm Plugins
 =================
 
+.. _scikit-bayesopt:
+
 Bayesian Optimizer
 ------------------
 
@@ -202,3 +223,83 @@ True if the target values' mean is expected to differ considerable from
 zero. When enabled, the normalization effectively modifies the GP's
 prior based on the data, which contradicts the likelihood principle;
 normalization is thus disabled per default.
+
+.. _parallel-strategies:
+
+Parallel Strategies
+===================
+
+A parallel strategy is a method to improve parallel optimization
+for sequential algorithms. Such algorithms can only observe
+trials that are completed and have a corresponding objective.
+To get around this, parallel strategies produces *lies*,
+noncompleted trials with fake objectives, which are then
+passed to a temporary copy of the algorithm that will suggest
+a new point. The temporary algorithm is then discarded.
+The original algorithm never obverses lies, and
+the temporary copy always observes lies that are based on
+most up-to-date data.
+The strategies will differ in how they assign objectives
+to the *lies*.
+
+By default, the strategy used is :ref:`MaxParallelStrategy`
+
+NoParallelStrategy
+------------------
+
+Does not return any lie. This is useful to benchmark parallel
+strategies and measure how they can help compared to no
+strategy.
+
+.. _StubParallelStrategy:
+
+StubParallelStrategy
+--------------------
+
+Assign to *lies* an objective of ``None`` so that
+non-completed trials are observed and identifiable by algorithms
+that can leverage parallel optimization.
+
+The value of the objective is customizable with ``stub_value``.
+
+.. code-block:: yaml
+
+    producer:
+      strategy:
+         StubParallelStrategy:
+            stub_value: 'custom value'
+
+.. _MaxParallelStrategy:
+
+MaxParallelStrategy
+-------------------
+
+Assigns to *lies* the best objective observed so far.
+
+The default value assigned to objective when less than 1 trial
+is completed is configurable with ``default_result``. It
+is ``float('inf')`` by default.
+
+.. code-block:: yaml
+
+    producer:
+      strategy:
+         MaxParallelStrategy:
+            default_result: 10000
+
+
+MeanParallelStrategy
+--------------------
+
+Assigns to *lies* the mean of all objectives observed so far.
+
+The default value assigned to objective when less than 2 trials
+are completed is configurable with ``default_result``. It
+is ``float('inf')`` by default.
+
+.. code-block:: yaml
+
+    producer:
+      strategy:
+         MeanParallelStrategy:
+            default_result: 0.5

src/orion/algo/asha.py

@@ -26,6 +26,12 @@
 Params: {params}
 """
 
+SPACE_ERROR = """
+ASHA cannot be used if space does contain a fidelity dimension.
+For more information on the configuration and usage of ASHA, see
+https://orion.readthedocs.io/en/develop/user/algorithms.html#asha
+"""
+
 
 class ASHA(BaseAlgorithm):
     """Asynchronous Successive Halving Algorithm
@@ -72,11 +78,16 @@ def __init__(self, space, seed=None, grace_period=None, max_resources=None,
                  reduction_factor=None, num_rungs=None, num_brackets=1):
         super(ASHA, self).__init__(
             space, seed=seed, max_resources=max_resources, grace_period=grace_period,
-            reduction_factor=reduction_factor, num_brackets=num_brackets)
+            reduction_factor=reduction_factor, num_rungs=num_rungs, num_brackets=num_brackets)
 
         self.trial_info = {}  # Stores Trial -> Bracket
 
-        fidelity_dim = space.values()[self.fidelity_index]
+        try:
+            fidelity_index = self.fidelity_index
+        except IndexError:
+            raise RuntimeError(SPACE_ERROR)
+
+        fidelity_dim = space.values()[fidelity_index]
 
         if grace_period is not None:
             logger.warning(
@@ -106,14 +117,15 @@ def __init__(self, space, seed=None, grace_period=None, max_resources=None,
             raise AttributeError("Reduction factor for ASHA needs to be at least 2.")
 
         if num_rungs is None:
-            num_rungs = numpy.log(max_resources / min_resources) / numpy.log(reduction_factor) + 1
+            num_rungs = int(numpy.log(max_resources / min_resources) /
+                            numpy.log(reduction_factor) + 1)
 
         self.num_rungs = num_rungs
 
         budgets = numpy.logspace(
             numpy.log(min_resources) / numpy.log(reduction_factor),
             numpy.log(max_resources) / numpy.log(reduction_factor),
-            num_rungs, base=reduction_factor)
+            num_rungs, base=reduction_factor).astype(int)
 
         # Tracks state for new trial add
         self.brackets = [
@@ -162,7 +174,7 @@ def suggest(self, num=1):
                 logger.debug('Promoting')
                 return [candidate]
 
-        if all(bracket.is_done for bracket in self.brackets):
+        if all(bracket.is_filled for bracket in self.brackets):
             logger.debug('All brackets are filled.')
             return None
 
@@ -178,7 +190,7 @@ def suggest(self, num=1):
 
         sizes = numpy.array([len(b.rungs) for b in self.brackets])
         probs = numpy.e**(sizes - sizes.max())
-        probs = numpy.array([prob * int(not bracket.is_done)
+        probs = numpy.array([prob * int(not bracket.is_filled)
                              for prob, bracket in zip(probs, self.brackets)])
         normalized = probs / probs.sum()
         idx = self.rng.choice(len(self.brackets), p=normalized)
@@ -298,14 +310,19 @@ def get_candidate(self, rung_id):
 
     @property
     def is_done(self):
+        """Return True, if the last rung is filled."""
+        return len(self.rungs[-1][1])
+
+    @property
+    def is_filled(self):
         """Return True, if the penultimate rung is filled."""
-        return self.is_filled(len(self.rungs) - 2) or len(self.rungs[-1][1])
+        return self.has_rung_filled(len(self.rungs) - 2)
 
-    def is_filled(self, rung_id):
+    def has_rung_filled(self, rung_id):
         """Return True, if the rung[rung_id] is filled."""
         n_rungs = len(self.rungs)
         n_trials = len(self.rungs[rung_id][1])
-        return n_trials >= (n_rungs - rung_id - 1) ** self.reduction_factor
+        return n_trials >= self.reduction_factor ** (n_rungs - rung_id - 1)
 
     def update_rungs(self):
         """Promote the first candidate that is found and return it
@@ -321,7 +338,7 @@ def update_rungs(self):
             Lookup for promotion in rung l + 1 contains trials of any status.
 
         """
-        if self.is_done and self.rungs[-1][1]:
+        if self.is_done:
             return None
 
         for rung_id in range(len(self.rungs) - 2, -1, -1):

src/orion/algo/space.py

@@ -670,6 +670,8 @@ class Fidelity(Dimension):
     ----------
     name : str
         Name of the dimension
+    default_value: int
+        Maximum of the fidelity interval.
 
     """
 
@@ -688,6 +690,11 @@ def __init__(self, name, low, high, base=2):
         self.prior = None
         self._prior_name = 'None'
 
+    @property
+    def default_value(self):
+        """Return `high`"""
+        return self.high
+
     def get_prior_string(self):
         """Build the string corresponding to current prior"""
         return 'fidelity({}, {}, {})'.format(self.low, self.high, self.base)
@@ -698,7 +705,7 @@ def validate(self):
 
     def sample(self, n_samples=1, seed=None):
         """Do not do anything."""
-        return ['fidelity']
+        return [self.high]
 
     def interval(self, alpha=1.0):
         """Do not do anything."""

src/orion/core/worker/__init__.py

@@ -25,7 +25,7 @@ def reserve_trial(experiment, producer):
     """Reserve a new trial, or produce and reserve a trial if none are available."""
     trial = experiment.reserve_trial(score_handle=producer.algorithm.score)
 
-    if trial is None:
+    if trial is None and not experiment.is_done:
         log.debug("#### Failed to pull a new trial from database.")
 
         log.debug("#### Fetch most recent completed trials and update algorithm.")
@@ -63,8 +63,9 @@ def workon(experiment, worker_trials=None):
         log.debug("#### Try to reserve a new trial to evaluate.")
         trial = reserve_trial(experiment, producer)
 
-        log.debug("#### Successfully reserved %s to evaluate. Consuming...", trial)
-        consumer.consume(trial)
+        if trial is not None:
+            log.debug("#### Successfully reserved %s to evaluate. Consuming...", trial)
+            consumer.consume(trial)
 
     stats = experiment.stats
 

src/orion/core/worker/producer.py

@@ -71,7 +71,7 @@ def produce(self):
         sampled_points = 0
 
         start = time.time()
-        while sampled_points < self.pool_size:
+        while sampled_points < self.pool_size and not self.algorithm.is_done:
             if time.time() - start > self.max_idle_time:
                 raise RuntimeError(
                     "Algorithm could not sample new points in less than {} seconds".format(

tests/conftest.py

@@ -53,7 +53,8 @@ def seed(self, seed):
     @property
     def state_dict(self):
         """Return a state dict that can be used to reset the state of the algorithm."""
-        return {'index': self._index, 'suggested': self._suggested, 'num': self._num}
+        return {'index': self._index, 'suggested': self._suggested, 'num': self._num,
+                'done': self.done}
 
     def set_state(self, state_dict):
         """Reset the state of the algorithm based on the given state_dict
@@ -63,6 +64,7 @@ def set_state(self, state_dict):
         self._index = state_dict['index']
         self._suggested = state_dict['suggested']
         self._num = state_dict['num']
+        self.done = state_dict['done']
 
     def suggest(self, num=1):
         """Suggest based on `value`."""

tests/functional/algos/asha_config.yaml

@@ -0,0 +1,21 @@
+name: demo_algo
+
+pool_size: 1
+max_trials: 100
+
+algorithms:
+  asha:
+    seed: 1
+    num_rungs: 4
+    num_brackets: 1
+    grace_period: null
+    max_resources: null
+    reduction_factor: null
+
+producer:
+  strategy: StubParallelStrategy
+
+database:
+  type: 'mongodb'
+  name: 'orion_test'
+  host: 'mongodb://user:pass@localhost'

tests/functional/algos/black_box.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Simple one dimensional example with noise level for a possible user's script."""
+import argparse
+import random
+
+from orion.client import report_results
+
+
+def function(x, noise):
+    """Evaluate partial information of a quadratic."""
+    z = (x - 34.56789) * random.gauss(0, noise)
+    return 4 * z**2 + 23.4, 8 * z
+
+
+def execute():
+    """Execute a simple pipeline as an example."""
+    # 1. Receive inputs as you want
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-x', type=float, required=True)
+    parser.add_argument('--fidelity', type=int, default=10)
+    inputs = parser.parse_args()
+
+    assert 0 <= inputs.fidelity <= 10
+
+    noise = (1 - inputs.fidelity / 10) + 0.0001
+
+    # 2. Perform computations
+    y, dy = function(inputs.x, noise)
+
+    # 3. Gather and report results
+    results = list()
+    results.append(dict(
+        name='example_objective',
+        type='objective',
+        value=y))
+    results.append(dict(
+        name='example_gradient',
+        type='gradient',
+        value=[dy]))
+
+    report_results(results)
+
+
+if __name__ == "__main__":
+    execute()