Add TPE algorithm into Orion

docs/src/code/algo.rst

@@ -16,3 +16,4 @@ TODO
    algo/random
    algo/hyperband
    algo/asha
+   algo/tpe

docs/src/code/algo/tpe.rst

@@ -0,0 +1,5 @@
+TPE Algorithm
+===================
+
+.. automodule:: orion.algo.tpe
+   :members:

docs/src/user/algorithms.rst

@@ -186,6 +186,71 @@ converging trials that do not lead to best results at convergence (stragglers).
 To overcome this, you can increase the number of brackets, which increases the amount of resources
 required for optimisation but decreases the bias towards stragglers. Default is 1.
 
+
+.. _tpe-algorithm:
+
+TPE
+---------
+
+`Tree-structured Parzen Estimator`_ (TPE) algorithm is one of Sequential Model-Based
+Global Optimization (SMBO) algorithms, which will build models to propose new points based
+on the historical observed trials.
+
+Instead of modeling p(y|x) like other SMBO algorithms, TPE models p(x|y) and p(y),
+and p(x|y) is modeled by transforming that generative process, replacing the distributions of
+the configuration prior with non-parametric densities.
+
+The TPE defines p(x|y) using two such densities l(x) and g(x) while l(x) is distribution of
+good points and g(x) is the distribution of bad points. New point candidates will be sampled
+with l(x) and Expected Improvement (EI) optimization scheme will be used to find the most
+promising point among the candidates.
+
+
+.. _Tree-structured Parzen Estimator:
+    https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf
+
+
+Configuration
+~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    algorithms:
+       tpe:
+          seed: null
+          n_initial_points: 20
+          n_ei_candidates: 25
+          gamma: 0.25
+          equal_weight: False
+          prior_weight: 1.0
+
+
+``seed``
+
+Seed to sample initial points and candidates points. Default is ``None``.
+
+``n_initial_points``
+
+Number of initial points randomly sampled. Default is ``20``.
+
+``n_ei_candidates``
+
+Number of candidates points sampled for ei compute. Default is ``24``.
+
+``gamma``
+
+Ratio to split the observed trials into good and bad distributions. Default is ``0.25``.
+
+``equal_weight``
+
+True to set equal weights for observed points. Default is ``False``.
+
+``seed``
+
+The weight given to the prior point of the input space. Default is ``1.0``.
+
+
+
 Algorithm Plugins
 =================
 

setup.py

@@ -45,6 +45,7 @@
             'random = orion.algo.random:Random',
             'asha = orion.algo.asha:ASHA',
             'hyperband = orion.algo.hyperband:Hyperband',
+            'tpe = orion.algo.tpe:TPE',
             ],
         'Storage': [
             'track = orion.storage.track:Track',

src/orion/algo/base.py

@@ -195,7 +195,7 @@ def observe(self, points, results):
             point_id = infer_trial_id(point)
 
             if point_id not in self._trials_info:
-                self._trials_info[point_id] = result
+                self._trials_info[point_id] = (point, result)
 
     @property
     def is_done(self):

src/orion/algo/hyperband.py

@@ -87,7 +87,7 @@ def display_budgets(budgets_tab, max_resources, reduction_factor):
     table_str += col_format_str.format(*col_sub_list)
     table_str += 'max resource={}, eta={}, trials number of one execution={}\n' \
         .format(max_resources, reduction_factor, total_trials)
-    logger.warning(table_str)
+    logger.debug(table_str)
 
 
 class Hyperband(BaseAlgorithm):

src/orion/algo/space.py

@@ -273,6 +273,11 @@ def type(self):
         """See `Dimension` attributes."""
         return self.__class__.__name__.lower()
 
+    @property
+    def prior_name(self):
+        """Return the name of the prior"""
+        return self._prior_name
+
     @property
     def shape(self):
         """Return the shape of dimension."""
@@ -533,6 +538,11 @@ def get_prior_string(self):
         prior_string = super(Integer, self).get_prior_string()
         return prior_string[:-1] + ', discrete=True)'
 
+    @property
+    def prior_name(self):
+        """Return the name of the prior"""
+        return "int_uniform"
+
     @property
     def cardinality(self):
         """Return the number of all the possible points from Integer `Dimension`"""
@@ -665,6 +675,11 @@ def get_prior_string(self):
 
         return 'choices({args})'.format(args=', '.join(args))
 
+    @property
+    def prior_name(self):
+        """Return the name of the prior"""
+        return "choices"
+
     def cast(self, point):
         """Cast a point to some category