TPE discrete-categorical-loguniform space support

src/orion/algo/tpe.py

@@ -11,7 +11,7 @@
 import logging
 
 import numpy
-from scipy.stats import lognorm, norm
+from scipy.stats import norm
 
 from orion.algo.base import BaseAlgorithm
 from orion.core.utils.points import flatten_dims, regroup_dims
@@ -186,11 +186,8 @@ def __init__(self, space, seed=None,
 
         for dimension in self.space.values():
 
-            if dimension.type not in ['real', 'integer', 'categorical']:
-                raise ValueError("TPE now only supports Real, Integer "
-                                 "and Categorical Dimension.")
-
-            if dimension.prior_name not in ['uniform', 'reciprocal', 'int_uniform', 'choices']:
+            if dimension.type != 'fidelity' and \
+                    dimension.prior_name not in ['uniform', 'reciprocal', 'int_uniform', 'choices']:
                 raise ValueError("TPE now only supports uniform, loguniform, uniform discrete "
                                  "and choices as prior.")
 
@@ -283,7 +280,8 @@ def suggest(self, num=1):
                                                        above_points[idx: idx + shape[0]],
                                                        self._sample_categorical_point)
                 else:
-                    raise NotImplementedError()
+                    # fidelity dimension
+                    points = dimension.sample(num)
 
                 if len(points) < shape[0]:
                     logger.warning('TPE failed to sample new point with configuration %s',
@@ -336,12 +334,10 @@ def _sample_real_point(self, dimension, below_points, above_points, is_log=False
         """Sample one value for real dimension based on the observed good and bad points"""
         low, high = dimension.interval()
         if is_log:
-            below_points = numpy.log(below_points)
-            above_points = numpy.log(above_points)
-
-            # scipy.stats loguniform
             low = numpy.log(low)
             high = numpy.log(high)
+            below_points = numpy.log(below_points)
+            above_points = numpy.log(above_points)
 
         below_mus, below_sigmas, below_weights = \
             adaptive_parzen_estimator(below_points, low, high, self.prior_weight,
@@ -351,15 +347,19 @@ def _sample_real_point(self, dimension, below_points, above_points, is_log=False
                                       self.equal_weight, flat_num=self.full_weight_num)
 
         gmm_sampler_below = GMMSampler(self, below_mus, below_sigmas,
-                                       low, high, below_weights, is_log=is_log)
+                                       low, high, below_weights)
         gmm_sampler_above = GMMSampler(self, above_mus, above_sigmas,
-                                       low, high, above_weights, is_log=is_log)
+                                       low, high, above_weights)
 
         candidate_points = gmm_sampler_below.sample(self.n_ei_candidates)
         if candidate_points:
             lik_blow = gmm_sampler_below.get_loglikelis(candidate_points)
             lik_above = gmm_sampler_above.get_loglikelis(candidate_points)
             new_point = compute_max_ei_point(candidate_points, lik_blow, lik_above)
+
+            if is_log:
+                new_point = numpy.exp(new_point)
+
             return new_point
 
         return None
@@ -452,29 +452,22 @@ class GMMSampler():
 
     """
 
-    def __init__(self, tpe, mus, sigmas, low, high, weights=None, is_log=False):
+    def __init__(self, tpe, mus, sigmas, low, high, weights=None):
         self.tpe = tpe
 
         self.mus = mus
         self.sigmas = sigmas
         self.low = low
         self.high = high
         self.weights = weights if weights is not None else len(mus) * [1.0 / len(mus)]
-        self.is_log = is_log
-        if is_log:
-            self.low = numpy.exp(low)
-            self.high = numpy.exp(high)
 
         self.pdfs = []
         self._build_mixture()
 
     def _build_mixture(self):
         """Build the Gaussian components in the GMM"""
         for mu, sigma in zip(self.mus, self.sigmas):
-            if self.is_log:
-                self.pdfs.append(lognorm(s=sigma, loc=0, scale=numpy.exp(mu)))
-            else:
-                self.pdfs.append(norm(mu, sigma))
+            self.pdfs.append(norm(mu, sigma))
 
     def sample(self, num=1):
         """Sample required number of points"""

tests/unittests/algo/test_tpe.py

@@ -4,7 +4,7 @@
 
 import numpy
 import pytest
-from scipy.stats import lognorm, norm
+from scipy.stats import norm
 
 from orion.algo.space import Categorical, Fidelity, Integer, Real, Space
 from orion.algo.tpe import adaptive_parzen_estimator, CategoricalSampler, \
@@ -278,11 +278,6 @@ def test_gmm_sampler_creation(self, tpe):
         assert len(gmm_sampler.weights) == 12
         assert len(gmm_sampler.pdfs) == 12
 
-        gmm_sampler = GMMSampler(tpe, mus, sigmas, -3, 3, is_log=True)
-
-        assert len(gmm_sampler.weights) == 12
-        assert len(gmm_sampler.pdfs) == 12
-
     def test_sample(self, tpe):
         """Test GMMSampler sample function"""
         mus = numpy.linspace(-3, 3, num=12, endpoint=False)
@@ -311,18 +306,6 @@ def test_sample(self, tpe):
         assert numpy.all(points >= -11)
         assert numpy.all(points < 9)
 
-        # loguniform
-        gmm_sampler = GMMSampler(tpe, mus, sigmas, -11, 9, weights, is_log=True)
-        points = gmm_sampler.sample(10000)
-        points = numpy.array(points)
-
-        bins = numpy.array([-11, -9, -7, -5, -3, -1, 1, 3, 5, 7, 9])
-        hist = numpy.histogram(points, bins=numpy.exp(bins))
-
-        assert numpy.all(hist[0].argsort() == numpy.array(weights).argsort())
-        assert numpy.all(points >= numpy.exp(-11))
-        assert numpy.all(points < numpy.exp(9))
-
     def test_get_loglikelis(self):
         """Test to get log likelis of points"""
         mus = numpy.linspace(-10, 10, num=10, endpoint=False)
@@ -361,26 +344,6 @@ def test_get_loglikelis(self):
         assert point_likeli == gmm_likeli
         assert len(likelis) == len(points)
 
-        # loguniform
-        gmm_sampler = GMMSampler(tpe, mus, sigmas, -11, 9, weights, is_log=True)
-
-        log_pdf = []
-        pdfs = []
-        for i in range(10):
-            pdfs.append(lognorm(s=sigmas[i], loc=0, scale=numpy.exp(mus[i])))
-        for pdf, weight in zip(pdfs, weights):
-            log_pdf.append(numpy.log(pdf.pdf(0) * weight))
-        point_likeli = numpy.log(numpy.sum(numpy.exp(log_pdf)))
-
-        points = numpy.random.uniform(-11, 9, 30)
-        points = numpy.insert(points, 10, 0)
-        likelis = gmm_sampler.get_loglikelis(points)
-
-        point_likeli = numpy.format_float_scientific(point_likeli, precision=10)
-        gmm_likeli = numpy.format_float_scientific(likelis[10], precision=10)
-        assert point_likeli == gmm_likeli
-        assert len(likelis) == len(points)
-
 
 class TestTPE():
     """Tests for the algo TPE."""
@@ -407,13 +370,16 @@ def test_set_state(self, tpe):
     def test_unsupported_space(self):
         """Test tpe only work for supported search space"""
         space = Space()
-        dim = Fidelity('epoch', 1, 9, 3)
-        space.register(dim)
-
-        with pytest.raises(ValueError) as ex:
-            TPE(space)
-
-        assert 'TPE now only supports Real, Integer and Categorical Dimension' in str(ex.value)
+        dim1 = Real('yolo1', 'uniform', -10, 10)
+        space.register(dim1)
+        dim2 = Real('yolo2', 'reciprocal', 10, 20)
+        space.register(dim2)
+        categories = ['a', 0.1, 2, 'c']
+        dim3 = Categorical('yolo3', categories)
+        space.register(dim3)
+        dim4 = Fidelity('epoch', 1, 9, 3)
+        space.register(dim4)
+        TPE(space)
 
         space = Space()
         dim = Real('yolo1', 'norm', 0.9)