From a16c8073bd22f571f5997681510eabb2ceca1aa9 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 26 Aug 2020 17:37:05 +0100
Subject: [PATCH 01/48] Added basic functionality and tests

---
 python/ray/tune/sample.py            | 205 +++++++++++++++++++++++++++
 python/ray/tune/tests/test_sample.py |  92 ++++++++++++
 python/requirements_tune.txt         |   1 +
 python/setup.py                      |   3 +-
 4 files changed, 299 insertions(+), 2 deletions(-)
 create mode 100644 python/ray/tune/tests/test_sample.py

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 4cac8572b417..3ba1bd6cfc83 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -1,11 +1,216 @@
 import logging
 import random
+from copy import copy
+from typing import Callable, Dict, Iterator, List, Optional, \
+    Sequence, \
+    Union
 
 import numpy as np
+from scipy import stats
 
 logger = logging.getLogger(__name__)
 
 
+class Domain:
+    _sampler = None
+
+    def set_sampler(self, sampler):
+        if self._sampler:
+            raise ValueError("You can only choose one sampler for parameter "
+                             "domains. Existing sampler for parameter {}: "
+                             "{}. Tried to add {}".format(
+                                 self.__class__.__name__, self._sampler,
+                                 sampler))
+        self._sampler = sampler
+
+    def sample(self, spec=None, size=1):
+        sampler = self._sampler
+        if not sampler:
+            sampler = Uniform()
+        return sampler.sample(self, spec=spec, size=size)
+
+
+class Float(Domain):
+    def __init__(self, min=float("-inf"), max=float("inf")):
+        self.min = min
+        self.max = max
+
+    def normal(self, mean=0., sd=1.):
+        new = copy(self)
+        new.set_sampler(Normal(mean, sd))
+        return new
+
+    def uniform(self):
+        if not self.min > float("-inf"):
+            raise ValueError(
+                "Uniform requires a minimum bound. Make sure to set the "
+                "`min` parameter of `Float()`.")
+        if not self.max < float("inf"):
+            raise ValueError(
+                "Uniform requires a maximum bound. Make sure to set the "
+                "`max` parameter of `Float()`.")
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+    def loguniform(self, base: int = 10):
+        if not self.min > 0:
+            raise ValueError(
+                "LogUniform requires a minimum bound greater than 0. "
+                "Make sure to set the `min` parameter of `Float()` correctly.")
+        if not 0 < self.max < float("inf"):
+            raise ValueError(
+                "LogUniform requires a minimum bound greater than 0. "
+                "Make sure to set the `max` parameter of `Float()` correctly.")
+        new = copy(self)
+        new.set_sampler(LogUniform(base))
+        return new
+
+
+class Integer(Domain):
+    def __init__(self, min, max):
+        self.min = min,
+        self.max = max
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Categorical(Domain):
+    def __init__(self, categories: Sequence):
+        self.categories = list(categories)
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Iterative(Domain):
+    def __init__(self, iterator: Iterator):
+        self.iterator = iterator
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Function(Domain):
+    def __init__(self, func: Callable):
+        self.func = func
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Sampler:
+    pass
+
+
+class Uniform(Sampler):
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        if isinstance(spec, list) and spec:
+            assert len(spec) == size, \
+                "Number of passed specs must match sample size"
+        elif isinstance(spec, dict) and spec:
+            assert size == 1, \
+                "Cannot sample the same parameter more than once for one spec"
+
+        if isinstance(domain, Float):
+            assert domain.min > float("-inf"), \
+                "Uniform needs a minimum bound"
+            assert 0 < domain.max < float("inf"), \
+                "Uniform needs a maximum bound"
+            return np.random.uniform(domain.min, domain.max, size=size)
+        elif isinstance(domain, Integer):
+            return np.random.randint(domain.min, domain.max, size=size)
+        elif isinstance(domain, Categorical):
+            choices = []
+            for i in range(size):
+                choices.append(random.choice(domain.categories))
+            if len(choices) == 1:
+                return choices[0]
+            return choices
+        elif isinstance(domain, Function):
+            items = []
+            for i in range(size):
+                this_spec = spec[i] if isinstance(spec, list) else spec
+                items.append(domain.func(this_spec))
+            if len(items) == 1:
+                return items[0]
+            return items
+        elif isinstance(domain, Iterative):
+            items = []
+            for i in range(size):
+                items.append(next(domain.iterator))
+            if len(items) == 1:
+                return items[0]
+            return items
+        else:
+            raise RuntimeError(
+                "Uniform sampler does not support parameters of type {}. "
+                "Allowed types: {}".format(
+                    domain.__class__.__name__,
+                    [Float, Integer, Categorical, Function, Iterative]))
+
+
+class LogUniform(Sampler):
+    def __init__(self, base: int = 10):
+        self.base = base
+        assert self.base > 0, "Base has to be strictly greater than 0"
+
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        if isinstance(domain, Float):
+            assert domain.min > 0, \
+                "LogUniform needs a minimum bound greater than 0"
+            assert 0 < domain.max < float("inf"), \
+                "LogUniform needs a maximum bound greater than 0"
+            logmin = np.log(domain.min) / np.log(self.base)
+            logmax = np.log(domain.max) / np.log(self.base)
+
+            return self.base**(np.random.uniform(logmin, logmax, size=size))
+        else:
+            raise RuntimeError(
+                "LogUniform sampler does not support parameters of type {}. "
+                "Allowed types: {}".format(domain.__class__.__name__, [Float]))
+
+
+class Normal(Sampler):
+    def __init__(self, mean: float = 0., sd: float = 0.):
+        self.mean = mean
+        self.sd = sd
+
+        assert self.sd > 0, "SD has to be strictly greater than 0"
+
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        if isinstance(domain, Float):
+            # Use a truncated normal to avoid oversampling border values
+            dist = stats.truncnorm(
+                (domain.min - self.mean) / self.sd,
+                (domain.max - self.mean) / self.sd,
+                loc=self.mean,
+                scale=self.sd)
+            return dist.rvs(size)
+        else:
+            raise ValueError(
+                "Normal sampler does not support parameters of type {}. "
+                "Allowed types: {}".format(domain.__class__.__name__, [Float]))
+
+
 class sample_from:
     """Specify that tune should sample configuration values from this function.
 
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
new file mode 100644
index 000000000000..12a7764081fc
--- /dev/null
+++ b/python/ray/tune/tests/test_sample.py
@@ -0,0 +1,92 @@
+import numpy as np
+import unittest
+
+from ray import tune
+
+
+class SearchSpaceTest(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testBoundedFloat(self):
+        bounded = tune.sample.Float(-4.2, 8.3)
+
+        # Don't allow to specify more than one sampler
+        with self.assertRaises(ValueError):
+            bounded.normal().uniform()
+
+        # Normal
+        samples = bounded.normal(-4, 2).sample(size=1000)
+        self.assertTrue(any(-4.2 < s < 8.3 for s in samples))
+        self.assertTrue(np.mean(samples) < -2)
+
+        # Uniform
+        samples = bounded.uniform().sample(size=1000)
+        self.assertTrue(any(-4.2 < s < 8.3 for s in samples))
+        self.assertFalse(np.mean(samples) < -2)
+
+        # Loguniform
+        with self.assertRaises(ValueError):
+            bounded.loguniform().sample(size=1000)
+
+        bounded_positive = tune.sample.Float(1e-4, 1e-1)
+        samples = bounded_positive.loguniform().sample(size=1000)
+        self.assertTrue(any(1e-4 < s < 1e-1 for s in samples))
+
+    def testUnboundedFloat(self):
+        unbounded = tune.sample.Float()
+
+        # Require min and max bounds for loguniform
+        with self.assertRaises(ValueError):
+            unbounded.loguniform()
+
+    def testBoundedInt(self):
+        bounded = tune.sample.Integer(-3, 12)
+
+        samples = bounded.uniform().sample(size=1000)
+        self.assertTrue(any(-3 <= s < 12 for s in samples))
+        self.assertFalse(np.mean(samples) < 2)
+
+    def testCategorical(self):
+        categories = [-2, -1, 0, 1, 2]
+        cat = tune.sample.Categorical(categories)
+
+        samples = cat.uniform().sample(size=1000)
+        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
+        self.assertTrue(all([c in samples for c in categories]))
+
+    def testIterative(self):
+        categories = [-2, -1, 0, 1, 2]
+
+        def test_iter():
+            for i in categories:
+                yield i
+
+        itr = tune.sample.Iterative(test_iter())
+        samples = itr.uniform().sample(size=5)
+        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
+        self.assertTrue(all([c in samples for c in categories]))
+
+        itr = tune.sample.Iterative(iter(categories))
+        samples = itr.uniform().sample(size=5)
+        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
+        self.assertTrue(all([c in samples for c in categories]))
+
+    def testFunction(self):
+        def sample(spec):
+            return np.random.uniform(-4, 4)
+
+        fnc = tune.sample.Function(sample)
+
+        samples = fnc.uniform().sample(size=1000)
+        self.assertTrue(any([-4 < s < 4 for s in samples]))
+        self.assertTrue(-2 < np.mean(samples) < 2)
+
+
+if __name__ == "__main__":
+    import pytest
+    import sys
+    sys.exit(pytest.main(["-v", __file__]))
diff --git a/python/requirements_tune.txt b/python/requirements_tune.txt
index 1123fc7ed879..b61dc62393a4 100644
--- a/python/requirements_tune.txt
+++ b/python/requirements_tune.txt
@@ -19,6 +19,7 @@ optuna
 pytest-remotedata>=0.3.1
 pytorch-lightning
 scikit-optimize
+scipy
 sigopt
 smart_open
 tensorflow_probability
diff --git a/python/setup.py b/python/setup.py
index d2a7c2872871..ff8b1177886d 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -111,7 +111,7 @@
 extras = {
     "debug": [],
     "serve": ["uvicorn", "flask", "requests"],
-    "tune": ["tabulate", "tensorboardX", "pandas"]
+    "tune": ["tabulate", "tensorboardX", "pandas", "scipy"]
 }
 
 extras["rllib"] = extras["tune"] + [
@@ -121,7 +121,6 @@
     "lz4",
     "opencv-python-headless<=4.3.0.36",
     "pyyaml",
-    "scipy",
 ]
 
 extras["streaming"] = []

From 6750060e8d7a17f6fbb67e43cf20e09af9fa594b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 27 Aug 2020 11:22:04 +0100
Subject: [PATCH 02/48] Feature parity with old tune search space config

---
 python/ray/tune/__init__.py                  |   4 +-
 python/ray/tune/experiment.py                |   4 +-
 python/ray/tune/sample.py                    | 139 +++++++++++--------
 python/ray/tune/schedulers/pbt.py            |  12 +-
 python/ray/tune/suggest/variant_generator.py |  30 ++--
 python/ray/tune/tests/test_var.py            |   8 +-
 6 files changed, 109 insertions(+), 88 deletions(-)

diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py
index 43a65c430a9d..13c82b5c8389 100644
--- a/python/ray/tune/__init__.py
+++ b/python/ray/tune/__init__.py
@@ -12,13 +12,13 @@
                               save_checkpoint, checkpoint_dir)
 from ray.tune.progress_reporter import (ProgressReporter, CLIReporter,
                                         JupyterNotebookReporter)
-from ray.tune.sample import (function, sample_from, uniform, choice, randint,
+from ray.tune.sample import (sample_from, uniform, choice, randint,
                              randn, loguniform)
 
 __all__ = [
     "Trainable", "DurableTrainable", "TuneError", "grid_search",
     "register_env", "register_trainable", "run", "run_experiments", "Stopper",
-    "EarlyStopping", "Experiment", "function", "sample_from", "track",
+    "EarlyStopping", "Experiment", "sample_from", "track",
     "uniform", "choice", "randint", "randn", "loguniform",
     "ExperimentAnalysis", "Analysis", "CLIReporter", "JupyterNotebookReporter",
     "ProgressReporter", "report", "get_trial_dir", "get_trial_name",
diff --git a/python/ray/tune/experiment.py b/python/ray/tune/experiment.py
index b6034eb67465..fd635ebb7a00 100644
--- a/python/ray/tune/experiment.py
+++ b/python/ray/tune/experiment.py
@@ -8,7 +8,7 @@
 from ray.tune.function_runner import detect_checkpoint_function
 from ray.tune.registry import register_trainable, get_trainable_cls
 from ray.tune.result import DEFAULT_RESULTS_DIR
-from ray.tune.sample import sample_from
+from ray.tune.sample import Domain
 from ray.tune.stopper import FunctionStopper, Stopper
 
 logger = logging.getLogger(__name__)
@@ -233,7 +233,7 @@ def register_if_needed(cls, run_object):
 
         if isinstance(run_object, str):
             return run_object
-        elif isinstance(run_object, sample_from):
+        elif isinstance(run_object, Domain):
             logger.warning("Not registering trainable. Resolving as variant.")
             return run_object
         elif isinstance(run_object, type) or callable(run_object):
diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 3ba1bd6cfc83..e5ffe1ed2694 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -29,6 +29,12 @@ def sample(self, spec=None, size=1):
             sampler = Uniform()
         return sampler.sample(self, spec=spec, size=size)
 
+    def is_grid(self):
+        return isinstance(self._sampler, Grid)
+
+    def is_function(self):
+        return False
+
 
 class Float(Domain):
     def __init__(self, min=float("-inf"), max=float("inf")):
@@ -87,6 +93,17 @@ def uniform(self):
         new.set_sampler(Uniform())
         return new
 
+    def grid(self):
+        new = copy(self)
+        new.set_sampler(Grid())
+        return new
+
+    def __len__(self):
+        return len(self.categories)
+
+    def __getitem__(self, item):
+        return self.categories[item]
+
 
 class Iterative(Domain):
     def __init__(self, iterator: Iterator):
@@ -107,6 +124,9 @@ def uniform(self):
         new.set_sampler(Uniform())
         return new
 
+    def is_function(self):
+        return True
+
 
 class Sampler:
     pass
@@ -129,9 +149,15 @@ def sample(self,
                 "Uniform needs a minimum bound"
             assert 0 < domain.max < float("inf"), \
                 "Uniform needs a maximum bound"
-            return np.random.uniform(domain.min, domain.max, size=size)
+            items = np.random.uniform(domain.min, domain.max, size=size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         elif isinstance(domain, Integer):
-            return np.random.randint(domain.min, domain.max, size=size)
+            items = np.random.randint(domain.min, domain.max, size=size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         elif isinstance(domain, Categorical):
             choices = []
             for i in range(size):
@@ -179,7 +205,10 @@ def sample(self,
             logmin = np.log(domain.min) / np.log(self.base)
             logmax = np.log(domain.max) / np.log(self.base)
 
-            return self.base**(np.random.uniform(logmin, logmax, size=size))
+            items = self.base**(np.random.uniform(logmin, logmax, size=size))
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         else:
             raise RuntimeError(
                 "LogUniform sampler does not support parameters of type {}. "
@@ -204,99 +233,91 @@ def sample(self,
                 (domain.max - self.mean) / self.sd,
                 loc=self.mean,
                 scale=self.sd)
-            return dist.rvs(size)
+            items = dist.rvs(size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         else:
             raise ValueError(
                 "Normal sampler does not support parameters of type {}. "
                 "Allowed types: {}".format(domain.__class__.__name__, [Float]))
 
 
-class sample_from:
+class Grid(Sampler):
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        return RuntimeError("Do not call `sample()` on grid.")
+
+
+def sample_from(func):
     """Specify that tune should sample configuration values from this function.
 
     Arguments:
         func: An callable function to draw a sample from.
     """
+    return Function(func)
 
-    def __init__(self, func):
-        self.func = func
-
-    def __str__(self):
-        return "tune.sample_from({})".format(str(self.func))
-
-    def __repr__(self):
-        return "tune.sample_from({})".format(repr(self.func))
-
-
-def function(func):
-    logger.warning(
-        "DeprecationWarning: wrapping {} with tune.function() is no "
-        "longer needed".format(func))
-    return func
 
+def uniform(min, max):
+    """Sample a float value uniformly between ``min`` and ``max``.
 
-class uniform(sample_from):
-    """Wraps tune.sample_from around ``np.random.uniform``.
-
-    ``tune.uniform(1, 10)`` is equivalent to
-    ``tune.sample_from(lambda _: np.random.uniform(1, 10))``
+    Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
+    ``np.random.uniform(1, 10))``
 
     """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: np.random.uniform(*args, **kwargs))
+    return Float(min, max).uniform()
 
 
-class loguniform(sample_from):
+def loguniform(min, max, base=10):
     """Sugar for sampling in different orders of magnitude.
 
     Args:
-        min_bound (float): Lower boundary of the output interval (1e-4)
-        max_bound (float): Upper boundary of the output interval (1e-2)
+        min (float): Lower boundary of the output interval (e.g. 1e-4)
+        max (float): Upper boundary of the output interval (e.g. 1e-2)
         base (float): Base of the log. Defaults to 10.
-    """
-
-    def __init__(self, min_bound, max_bound, base=10):
-        logmin = np.log(min_bound) / np.log(base)
-        logmax = np.log(max_bound) / np.log(base)
 
-        def apply_log(_):
-            return base**(np.random.uniform(logmin, logmax))
-
-        super().__init__(apply_log)
+    """
+    return Float(min, max).loguniform(base)
 
 
-class choice(sample_from):
-    """Wraps tune.sample_from around ``random.choice``.
+def choice(categories):
+    """Sample a categorical value.
 
-    ``tune.choice([1, 2])`` is equivalent to
-    ``tune.sample_from(lambda _: random.choice([1, 2]))``
+    Sampling from ``tune.choice([1, 2])`` is equivalent to sampling from
+    ``random.choice([1, 2])``
 
     """
+    return Categorical(categories).uniform()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: random.choice(*args, **kwargs))
 
+def randint(min, max):
+    """Sample an integer value uniformly between ``min`` and ``max``.
 
-class randint(sample_from):
-    """Wraps tune.sample_from around ``np.random.randint``.
+    ``min`` is inclusive, ``max`` is exlcusive.
 
-    ``tune.randint(10)`` is equivalent to
-    ``tune.sample_from(lambda _: np.random.randint(10))``
+    Sampling from ``tune.randint(10)`` is equivalent to sampling from
+    ``np.random.randint(10)``
 
     """
+    return Integer(min, max).uniform()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: np.random.randint(*args, **kwargs))
 
+def randn(mean: float = 0.,
+          sd: float = 1.,
+          min: float = float("-inf"),
+          max: float = float("inf")):
+    """Sample a float value normally with ``mean`` and ``sd``.
 
-class randn(sample_from):
-    """Wraps tune.sample_from around ``np.random.randn``.
+    Will truncate the normal distribution at ``min`` and ``max`` to avoid
+    oversampling the border regions.
 
-    ``tune.randn(10)`` is equivalent to
-    ``tune.sample_from(lambda _: np.random.randn(10))``
+    Args:
+        mean (float): Mean of the normal distribution. Defaults to 0.
+        sd (float): SD of the normal distribution. Defaults to 1.
+        min (float): Minimum bound. Defaults to -inf.
+        max (float): Maximum bound. Defaults to inf.
 
     """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: np.random.randn(*args, **kwargs))
+    return Float(min, max).normal(mean, sd)
diff --git a/python/ray/tune/schedulers/pbt.py b/python/ray/tune/schedulers/pbt.py
index 588c247bde46..40c71ae363a1 100644
--- a/python/ray/tune/schedulers/pbt.py
+++ b/python/ray/tune/schedulers/pbt.py
@@ -9,7 +9,7 @@
 from ray.tune.error import TuneError
 from ray.tune.result import TRAINING_ITERATION
 from ray.tune.logger import _SafeFallbackEncoder
-from ray.tune.sample import sample_from
+from ray.tune.sample import Domain, sample_from
 from ray.tune.schedulers import FIFOScheduler, TrialScheduler
 from ray.tune.suggest.variant_generator import format_vars
 from ray.tune.trial import Trial, Checkpoint
@@ -66,8 +66,8 @@ def explore(config, mutations, resample_probability, custom_explore_fn):
                     distribution.index(config[key]) + 1)]
         else:
             if random.random() < resample_probability:
-                new_config[key] = distribution.func(None) if isinstance(
-                    distribution, sample_from) else distribution()
+                new_config[key] = distribution.sample(None) if isinstance(
+                    distribution, Domain) else distribution()
             elif random.random() > 0.5:
                 new_config[key] = config[key] * 1.2
             else:
@@ -94,8 +94,8 @@ def fill_config(config, attr, search_space):
     """Add attr to config by sampling from search_space."""
     if callable(search_space):
         config[attr] = search_space()
-    elif isinstance(search_space, sample_from):
-        config[attr] = search_space.func(None)
+    elif isinstance(search_space, Domain):
+        config[attr] = search_space.sample(None)
     elif isinstance(search_space, list):
         config[attr] = random.choice(search_space)
     elif isinstance(search_space, dict):
@@ -218,7 +218,7 @@ def __init__(self,
                  require_attrs=True):
         for value in hyperparam_mutations.values():
             if not (isinstance(value,
-                               (list, dict, sample_from)) or callable(value)):
+                               (list, dict, Domain)) or callable(value)):
                 raise TypeError("`hyperparam_mutation` values must be either "
                                 "a List, Dict, a tune search space object, or "
                                 "callable.")
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index e772ffb4933c..4d908104eefa 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -4,7 +4,7 @@
 import random
 
 from ray.tune import TuneError
-from ray.tune.sample import sample_from
+from ray.tune.sample import Categorical, Domain, Function
 
 logger = logging.getLogger(__name__)
 
@@ -123,17 +123,17 @@ def _generate_variants(spec):
         return
 
     grid_vars = []
-    lambda_vars = []
+    domain_vars = []
     for path, value in unresolved.items():
-        if callable(value):
-            lambda_vars.append((path, value))
-        else:
+        if value.is_grid():
             grid_vars.append((path, value))
+        else:
+            domain_vars.append((path, value))
     grid_vars.sort()
 
     grid_search = _grid_search_generator(spec, grid_vars)
     for resolved_spec in grid_search:
-        resolved_vars = _resolve_lambda_vars(resolved_spec, lambda_vars)
+        resolved_vars = _resolve_domain_vars(resolved_spec, domain_vars)
         for resolved, spec in _generate_variants(resolved_spec):
             for path, value in grid_vars:
                 resolved_vars[path] = _get_value(spec, path)
@@ -160,16 +160,16 @@ def _get_value(spec, path):
     return spec
 
 
-def _resolve_lambda_vars(spec, lambda_vars):
+def _resolve_domain_vars(spec, domain_vars):
     resolved = {}
     error = True
     num_passes = 0
     while error and num_passes < _MAX_RESOLUTION_PASSES:
         num_passes += 1
         error = False
-        for path, fn in lambda_vars:
+        for path, fn in domain_vars:
             try:
-                value = fn(_UnresolvedAccessGuard(spec))
+                value = fn.sample(_UnresolvedAccessGuard(spec))
             except RecursiveDependencyError as e:
                 error = e
             except Exception:
@@ -217,13 +217,13 @@ def _is_resolved(v):
 
 
 def _try_resolve(v):
-    if isinstance(v, sample_from):
-        # Function to sample from
-        return False, v.func
+    if isinstance(v, Domain):
+        # Domain to sample from
+        return False, v
     elif isinstance(v, dict) and len(v) == 1 and "eval" in v:
         # Lambda function in eval syntax
-        return False, lambda spec: eval(
-            v["eval"], _STANDARD_IMPORTS, {"spec": spec})
+        return False, Function(
+            lambda spec: eval(v["eval"], _STANDARD_IMPORTS, {"spec": spec}))
     elif isinstance(v, dict) and len(v) == 1 and "grid_search" in v:
         # Grid search values
         grid_values = v["grid_search"]
@@ -231,7 +231,7 @@ def _try_resolve(v):
             raise TuneError(
                 "Grid search expected list of values, got: {}".format(
                     grid_values))
-        return False, grid_values
+        return False, Categorical(grid_values).grid()
     return True, v
 
 
diff --git a/python/ray/tune/tests/test_var.py b/python/ray/tune/tests/test_var.py
index 3df8e7f16edb..082a9fe0345c 100644
--- a/python/ray/tune/tests/test_var.py
+++ b/python/ray/tune/tests/test_var.py
@@ -263,13 +263,13 @@ def testNestedValues(self):
         })
 
     def testLogUniform(self):
-        sampler = tune.loguniform(1e-10, 1e-1).func
-        results = [sampler(None) for i in range(1000)]
+        sampler = tune.loguniform(1e-10, 1e-1)
+        results = sampler.sample(None, 1000)
         assert abs(np.log(min(results)) / np.log(10) - -10) < 0.1
         assert abs(np.log(max(results)) / np.log(10) - -1) < 0.1
 
-        sampler_e = tune.loguniform(np.e**-4, np.e, base=np.e).func
-        results_e = [sampler_e(None) for i in range(1000)]
+        sampler_e = tune.loguniform(np.e**-4, np.e, base=np.e)
+        results_e = sampler_e.sample(None, 1000)
         assert abs(np.log(min(results_e)) - -4) < 0.1
         assert abs(np.log(max(results_e)) - 1) < 0.1
 

From aaaab736d2e15e907a343c3058e832d51e346617 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 27 Aug 2020 18:27:21 +0100
Subject: [PATCH 03/48] Convert Optuna search spaces

---
 python/ray/tune/__init__.py                  | 14 +++----
 python/ray/tune/sample.py                    | 20 +++++----
 python/ray/tune/suggest/optuna.py            | 43 ++++++++++++++++++++
 python/ray/tune/suggest/variant_generator.py | 30 +++++++++-----
 python/ray/tune/tests/test_sample.py         | 30 ++++++++++++++
 5 files changed, 113 insertions(+), 24 deletions(-)

diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py
index 13c82b5c8389..6312fe8986dc 100644
--- a/python/ray/tune/__init__.py
+++ b/python/ray/tune/__init__.py
@@ -12,15 +12,15 @@
                               save_checkpoint, checkpoint_dir)
 from ray.tune.progress_reporter import (ProgressReporter, CLIReporter,
                                         JupyterNotebookReporter)
-from ray.tune.sample import (sample_from, uniform, choice, randint,
-                             randn, loguniform)
+from ray.tune.sample import (sample_from, uniform, choice, randint, randn,
+                             loguniform)
 
 __all__ = [
     "Trainable", "DurableTrainable", "TuneError", "grid_search",
     "register_env", "register_trainable", "run", "run_experiments", "Stopper",
-    "EarlyStopping", "Experiment", "sample_from", "track",
-    "uniform", "choice", "randint", "randn", "loguniform",
-    "ExperimentAnalysis", "Analysis", "CLIReporter", "JupyterNotebookReporter",
-    "ProgressReporter", "report", "get_trial_dir", "get_trial_name",
-    "get_trial_id", "make_checkpoint_dir", "save_checkpoint", "checkpoint_dir"
+    "EarlyStopping", "Experiment", "sample_from", "track", "uniform", "choice",
+    "randint", "randn", "loguniform", "ExperimentAnalysis", "Analysis",
+    "CLIReporter", "JupyterNotebookReporter", "ProgressReporter", "report",
+    "get_trial_dir", "get_trial_name", "get_trial_id", "make_checkpoint_dir",
+    "save_checkpoint", "checkpoint_dir"
 ]
diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index e5ffe1ed2694..03f0854be9fa 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -12,25 +12,31 @@
 
 
 class Domain:
-    _sampler = None
+    sampler = None
 
     def set_sampler(self, sampler):
-        if self._sampler:
+        if self.sampler:
             raise ValueError("You can only choose one sampler for parameter "
                              "domains. Existing sampler for parameter {}: "
                              "{}. Tried to add {}".format(
-                                 self.__class__.__name__, self._sampler,
+                                 self.__class__.__name__, self.sampler,
                                  sampler))
-        self._sampler = sampler
+        self.sampler = sampler
+
+    def has_sampler(self, cls):
+        sampler = self.sampler
+        if not sampler:
+            sampler = Uniform()
+        return isinstance(sampler, cls)
 
     def sample(self, spec=None, size=1):
-        sampler = self._sampler
+        sampler = self.sampler
         if not sampler:
             sampler = Uniform()
         return sampler.sample(self, spec=spec, size=size)
 
     def is_grid(self):
-        return isinstance(self._sampler, Grid)
+        return isinstance(self.sampler, Grid)
 
     def is_function(self):
         return False
@@ -75,7 +81,7 @@ def loguniform(self, base: int = 10):
 
 class Integer(Domain):
     def __init__(self, min, max):
-        self.min = min,
+        self.min = min
         self.max = max
 
     def uniform(self):
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 51d3efc14ee7..0a81822a9ebc 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -1,7 +1,11 @@
 import logging
 import pickle
+from typing import Dict
 
 from ray.tune.result import TRAINING_ITERATION
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, Uniform
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
 
 try:
     import optuna as ot
@@ -147,3 +151,42 @@ def restore(self, checkpoint_path):
             save_object = pickle.load(inputFile)
         self._storage, self._pruner, self._sampler, \
             self._ot_trials, self._ot_study = save_object
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec)
+        domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return []
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an Optuna search space.")
+
+        values = []
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            if isinstance(domain, Float) and domain.has_sampler(LogUniform):
+                value = param.suggest_loguniform(par, domain.min, domain.max)
+            elif isinstance(domain, Float) and domain.has_sampler(Uniform):
+                value = param.suggest_uniform(par, domain.min, domain.max)
+            elif isinstance(domain,
+                            Integer) and domain.has_sampler(LogUniform):
+                value = param.suggest_int(
+                    par, domain.min, domain.max, log=True)
+            elif isinstance(domain, Integer) and domain.has_sampler(Uniform):
+                value = param.suggest_int(
+                    par, domain.min, domain.max, log=False)
+            elif isinstance(domain,
+                            Categorical) and domain.has_sampler(Uniform):
+                value = param.suggest_categorical(par, domain.categories)
+            else:
+                raise ValueError(
+                    "Optuna search does not support parameters of type "
+                    "{} with samplers of type {}".format(
+                        type(domain), type(domain.sampler)))
+            values.append(value)
+
+        return values
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index 4d908104eefa..00040ce0b599 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -115,12 +115,10 @@ def _clean_value(value):
         return str(value).replace("/", "_")
 
 
-def _generate_variants(spec):
-    spec = copy.deepcopy(spec)
+def parse_spec_vars(spec):
     unresolved = _unresolved_values(spec)
     if not unresolved:
-        yield {}, spec
-        return
+        return [], []
 
     grid_vars = []
     domain_vars = []
@@ -131,6 +129,17 @@ def _generate_variants(spec):
             domain_vars.append((path, value))
     grid_vars.sort()
 
+    return domain_vars, grid_vars
+
+
+def _generate_variants(spec):
+    spec = copy.deepcopy(spec)
+    domain_vars, grid_vars = parse_spec_vars(spec)
+
+    if not domain_vars and not grid_vars:
+        yield {}, spec
+        return
+
     grid_search = _grid_search_generator(spec, grid_vars)
     for resolved_spec in grid_search:
         resolved_vars = _resolve_domain_vars(resolved_spec, domain_vars)
@@ -148,7 +157,7 @@ def _generate_variants(spec):
             yield resolved_vars, spec
 
 
-def _assign_value(spec, path, value):
+def assign_value(spec, path, value):
     for k in path[:-1]:
         spec = spec[k]
     spec[path[-1]] = value
@@ -167,16 +176,17 @@ def _resolve_domain_vars(spec, domain_vars):
     while error and num_passes < _MAX_RESOLUTION_PASSES:
         num_passes += 1
         error = False
-        for path, fn in domain_vars:
+        for path, domain in domain_vars:
             try:
-                value = fn.sample(_UnresolvedAccessGuard(spec))
+                value = domain.sample(_UnresolvedAccessGuard(spec))
             except RecursiveDependencyError as e:
                 error = e
             except Exception:
                 raise ValueError(
-                    "Failed to evaluate expression: {}: {}".format(path, fn))
+                    "Failed to evaluate expression: {}: {}".format(
+                        path, domain))
             else:
-                _assign_value(spec, path, value)
+                assign_value(spec, path, value)
                 resolved[path] = value
     if error:
         raise error
@@ -203,7 +213,7 @@ def increment(i):
     while value_indices[-1] < len(grid_vars[-1][1]):
         spec = copy.deepcopy(unresolved_spec)
         for i, (path, values) in enumerate(grid_vars):
-            _assign_value(spec, path, values[value_indices[i]])
+            assign_value(spec, path, values[value_indices[i]])
         yield spec
         if grid_vars:
             done = increment(0)
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 12a7764081fc..6423df6ec092 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -85,6 +85,36 @@ def sample(spec):
         self.assertTrue(any([-4 < s < 4 for s in samples]))
         self.assertTrue(-2 < np.mean(samples) < 2)
 
+    def testConvertOptuna(self):
+        from ray.tune.suggest.optuna import OptunaSearch, param
+        from optuna.samplers import RandomSampler
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = OptunaSearch.convert_search_space(config)
+        optuna_config = [
+            param.suggest_categorical("a", [2, 3, 4]),
+            param.suggest_int("b/x", 0, 5),
+            param.suggest_loguniform("b/z", 1e-4, 1e-2)
+        ]
+
+        sampler1 = RandomSampler(seed=1234)
+        searcher1 = OptunaSearch(space=converted_config, sampler=sampler1)
+
+        sampler2 = RandomSampler(seed=1234)
+        searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2)
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+
 
 if __name__ == "__main__":
     import pytest

From 546e9ba7469388473c51ba3879745fe96a62308b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 27 Aug 2020 18:58:42 +0100
Subject: [PATCH 04/48] Introduced quantized values

---
 python/ray/tune/sample.py            | 91 ++++++++++++++++++++++++++--
 python/ray/tune/tests/test_sample.py |  8 +++
 2 files changed, 95 insertions(+), 4 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 03f0854be9fa..f8f0242dd9e6 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -1,6 +1,7 @@
 import logging
 import random
 from copy import copy
+from numbers import Number
 from typing import Callable, Dict, Iterator, List, Optional, \
     Sequence, \
     Union
@@ -14,8 +15,8 @@
 class Domain:
     sampler = None
 
-    def set_sampler(self, sampler):
-        if self.sampler:
+    def set_sampler(self, sampler, allow_override=False):
+        if self.sampler and not allow_override:
             raise ValueError("You can only choose one sampler for parameter "
                              "domains. Existing sampler for parameter {}: "
                              "{}. Tried to add {}".format(
@@ -47,6 +48,11 @@ def __init__(self, min=float("-inf"), max=float("inf")):
         self.min = min
         self.max = max
 
+    def quantized(self, q: Number):
+        new = copy(self)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        return new
+
     def normal(self, mean=0., sd=1.):
         new = copy(self)
         new.set_sampler(Normal(mean, sd))
@@ -84,6 +90,11 @@ def __init__(self, min, max):
         self.min = min
         self.max = max
 
+    def quantized(self, q: Number):
+        new = copy(self)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        return new
+
     def uniform(self):
         new = copy(self)
         new.set_sampler(Uniform())
@@ -135,7 +146,11 @@ def is_function(self):
 
 
 class Sampler:
-    pass
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        raise NotImplementedError
 
 
 class Uniform(Sampler):
@@ -249,6 +264,22 @@ def sample(self,
                 "Allowed types: {}".format(domain.__class__.__name__, [Float]))
 
 
+class Quantized(Sampler):
+    def __init__(self, sampler: Sampler, q: Number):
+        self.sampler = sampler
+        self.q = q
+
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        values = self.sampler.sample(domain, spec, size)
+        quantized = np.round(np.divide(values, self.q)) * self.q
+        if len(quantized) == 1:
+            return quantized[0]
+        return list(quantized)
+
+
 class Grid(Sampler):
     def sample(self,
                domain: Domain,
@@ -276,18 +307,46 @@ def uniform(min, max):
     return Float(min, max).uniform()
 
 
+def quniform(min, max, q):
+    """Sample a quantized float value uniformly between ``min`` and ``max``.
+
+    Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
+    ``np.random.uniform(1, 10))``
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    """
+    return Float(min, max).uniform().quantized(q)
+
+
 def loguniform(min, max, base=10):
     """Sugar for sampling in different orders of magnitude.
 
     Args:
         min (float): Lower boundary of the output interval (e.g. 1e-4)
         max (float): Upper boundary of the output interval (e.g. 1e-2)
-        base (float): Base of the log. Defaults to 10.
+        base (int): Base of the log. Defaults to 10.
 
     """
     return Float(min, max).loguniform(base)
 
 
+def qloguniform(min, max, q, base=10):
+    """Sugar for sampling in different orders of magnitude.
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    Args:
+        min (float): Lower boundary of the output interval (e.g. 1e-4)
+        max (float): Upper boundary of the output interval (e.g. 1e-2)
+        q (float): Quantization number. The result will be rounded to an
+            integer increment of this value.
+        base (int): Base of the log. Defaults to 10.
+
+    """
+    return Float(min, max).loguniform(base).quantized(q)
+
+
 def choice(categories):
     """Sample a categorical value.
 
@@ -327,3 +386,27 @@ def randn(mean: float = 0.,
 
     """
     return Float(min, max).normal(mean, sd)
+
+
+def qrandn(q: float,
+           mean: float = 0.,
+           sd: float = 1.,
+           min: float = float("-inf"),
+           max: float = float("inf")):
+    """Sample a float value normally with ``mean`` and ``sd``.
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    Will truncate the normal distribution at ``min`` and ``max`` to avoid
+    oversampling the border regions.
+
+    Args:
+        q (float): Quantization number. The result will be rounded to an
+            integer increment of this value.
+        mean (float): Mean of the normal distribution. Defaults to 0.
+        sd (float): SD of the normal distribution. Defaults to 1.
+        min (float): Minimum bound. Defaults to -inf.
+        max (float): Maximum bound. Defaults to inf.
+
+    """
+    return Float(min, max).normal(mean, sd).quantized(q)
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 6423df6ec092..b2c42602587a 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -85,6 +85,14 @@ def sample(spec):
         self.assertTrue(any([-4 < s < 4 for s in samples]))
         self.assertTrue(-2 < np.mean(samples) < 2)
 
+    def testQuantized(self):
+        bounded_positive = tune.sample.Float(1e-4, 1e-1)
+        samples = bounded_positive.loguniform().quantized(5e-4).sample(size=10)
+
+        for sample in samples:
+            factor = sample / 5e-4
+            self.assertAlmostEqual(factor, round(factor), places=10)
+
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
         from optuna.samplers import RandomSampler

From 18535a6621950831adb09f39a4980a92ab01759b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 16:31:33 +0100
Subject: [PATCH 05/48] Updated Optuna resolving

---
 python/ray/tune/sample.py            | 12 ++--
 python/ray/tune/suggest/optuna.py    | 82 +++++++++++++++++++---------
 python/ray/tune/tests/test_sample.py | 10 ++--
 3 files changed, 68 insertions(+), 36 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index f8f0242dd9e6..65459f575b7b 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -24,16 +24,14 @@ def set_sampler(self, sampler, allow_override=False):
                                  sampler))
         self.sampler = sampler
 
-    def has_sampler(self, cls):
+    def get_sampler(self):
         sampler = self.sampler
         if not sampler:
             sampler = Uniform()
-        return isinstance(sampler, cls)
+        return sampler
 
     def sample(self, spec=None, size=1):
-        sampler = self.sampler
-        if not sampler:
-            sampler = Uniform()
+        sampler = self.get_sampler()
         return sampler.sample(self, spec=spec, size=size)
 
     def is_grid(self):
@@ -50,7 +48,7 @@ def __init__(self, min=float("-inf"), max=float("inf")):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
         return new
 
     def normal(self, mean=0., sd=1.):
@@ -92,7 +90,7 @@ def __init__(self, min, max):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
         return new
 
     def uniform(self):
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 0a81822a9ebc..cde4bb6a77a1 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -1,10 +1,12 @@
+import copy
 import logging
 import pickle
 from typing import Dict
 
 from ray.tune.result import TRAINING_ITERATION
-from ray.tune.sample import Categorical, Float, Integer, LogUniform, Uniform
-from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
+    Quantized, Uniform
+from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars
 from ray.tune.utils import flatten_dict
 
 try:
@@ -57,6 +59,10 @@ class OptunaSearch(Searcher):
             minimizing or maximizing the metric attribute.
         sampler (optuna.samplers.BaseSampler): Optuna sampler used to
             draw hyperparameter configurations. Defaults to ``TPESampler``.
+        config (dict): Base config dict that gets overwritten by the Optuna
+            sampling and is returned to each Tune trial. This could e.g.
+            contain static variables or configurations that should be passed
+            to each trial.
 
     Example:
 
@@ -84,6 +90,7 @@ def __init__(
             metric="episode_reward_mean",
             mode="max",
             sampler=None,
+            config=None,
     ):
         assert ot is not None, (
             "Optuna must be installed! Run `pip install optuna`.")
@@ -95,6 +102,8 @@ def __init__(
 
         self._space = space
 
+        self._config = config or {}
+
         self._study_name = "optuna"  # Fixed study name for in-memory storage
         self._sampler = sampler or ot.samplers.TPESampler()
         assert isinstance(self._sampler, ot.samplers.BaseSampler), \
@@ -120,10 +129,11 @@ def suggest(self, trial_id):
             self._ot_trials[trial_id] = ot.trial.Trial(self._ot_study,
                                                        ot_trial_id)
         ot_trial = self._ot_trials[trial_id]
-        params = {}
+        params = copy.copy(self._config)
         for (fn, args, kwargs) in self._space:
             param_name = args[0] if len(args) > 0 else kwargs["name"]
-            params[param_name] = getattr(ot_trial, fn)(*args, **kwargs)
+            value = getattr(ot_trial, fn)(*args, **kwargs)  # Call Optuna trial
+            assign_value(params, param_name.split("/"), value)
         return params
 
     def on_trial_result(self, trial_id, result):
@@ -165,28 +175,50 @@ def convert_search_space(spec: Dict):
                 "Grid search parameters cannot be automatically converted "
                 "to an Optuna search space.")
 
+        def resolve_value(par, domain):
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        logger.warning(
+                            "Optuna does not support both quantization and "
+                            "sampling from LogUniform. Dropped quantization.")
+                    return param.suggest_loguniform(par, domain.min,
+                                                    domain.max)
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return param.suggest_discrete_uniform(
+                            par, domain.min, domain.max, quantize)
+                    return param.suggest_uniform(par, domain.min, domain.max)
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        logger.warning(
+                            "Optuna does not support both quantization and "
+                            "sampling from LogUniform. Dropped quantization.")
+                    return param.suggest_int(
+                        par, domain.min, domain.max, log=True)
+                elif isinstance(sampler, Uniform):
+                    return param.suggest_int(
+                        par, domain.min, domain.max, step=quantize or 1)
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return param.suggest_categorical(par, domain.categories)
+
+            raise ValueError(
+                "Optuna search does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__,
+                    type(domain.sampler).__name__))
+
         values = []
         for path, domain in domain_vars:
             par = "/".join(path)
-            if isinstance(domain, Float) and domain.has_sampler(LogUniform):
-                value = param.suggest_loguniform(par, domain.min, domain.max)
-            elif isinstance(domain, Float) and domain.has_sampler(Uniform):
-                value = param.suggest_uniform(par, domain.min, domain.max)
-            elif isinstance(domain,
-                            Integer) and domain.has_sampler(LogUniform):
-                value = param.suggest_int(
-                    par, domain.min, domain.max, log=True)
-            elif isinstance(domain, Integer) and domain.has_sampler(Uniform):
-                value = param.suggest_int(
-                    par, domain.min, domain.max, log=False)
-            elif isinstance(domain,
-                            Categorical) and domain.has_sampler(Uniform):
-                value = param.suggest_categorical(par, domain.categories)
-            else:
-                raise ValueError(
-                    "Optuna search does not support parameters of type "
-                    "{} with samplers of type {}".format(
-                        type(domain), type(domain.sampler)))
-            values.append(value)
-
+            values.append(resolve_value(par, domain))
         return values
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index b2c42602587a..1c03dbf7309b 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -100,7 +100,7 @@ def testConvertOptuna(self):
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
-                "x": tune.sample.Integer(0, 5),
+                "x": tune.sample.Integer(0, 5).quantized(2),
                 "y": 4,
                 "z": tune.sample.Float(1e-4, 1e-2).loguniform()
             }
@@ -108,15 +108,17 @@ def testConvertOptuna(self):
         converted_config = OptunaSearch.convert_search_space(config)
         optuna_config = [
             param.suggest_categorical("a", [2, 3, 4]),
-            param.suggest_int("b/x", 0, 5),
+            param.suggest_int("b/x", 0, 5, 2),
             param.suggest_loguniform("b/z", 1e-4, 1e-2)
         ]
 
         sampler1 = RandomSampler(seed=1234)
-        searcher1 = OptunaSearch(space=converted_config, sampler=sampler1)
+        searcher1 = OptunaSearch(
+            space=converted_config, sampler=sampler1, config=config)
 
         sampler2 = RandomSampler(seed=1234)
-        searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2)
+        searcher2 = OptunaSearch(
+            space=optuna_config, sampler=sampler2, config=config)
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")

From 3ab491c895e8bed40585e4a11fc1ec3a822aea5f Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 17:06:09 +0100
Subject: [PATCH 06/48] Added HyperOpt search space conversion

---
 python/ray/tune/suggest/hyperopt.py  | 81 ++++++++++++++++++++++++++++
 python/ray/tune/tests/test_sample.py | 42 +++++++++++++++
 2 files changed, 123 insertions(+)

diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 5bdb1cbbdd08..7c1eacf8a237 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -1,8 +1,16 @@
+from typing import Dict
+
 import numpy as np
 import copy
 import logging
 from functools import partial
 import pickle
+
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, Normal, \
+    Quantized, \
+    Uniform
+from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars
+
 try:
     hyperopt_logger = logging.getLogger("hyperopt")
     hyperopt_logger.setLevel(logging.WARNING)
@@ -235,3 +243,76 @@ def restore(self, checkpoint_path):
             self.rstate.set_state(trials_object[1])
         else:
             self.set_state(trials_object)
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = copy.deepcopy(spec)
+        domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return []
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a HyperOpt search space.")
+
+        def resolve_value(par, domain):
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        return hpo.hp.qloguniform(par, domain.min, domain.max,
+                                                  quantize)
+                    return hpo.hp.loguniform(par, np.log(domain.min),
+                                             np.log(domain.max))
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return hpo.hp.quniform(par, domain.min, domain.max,
+                                               quantize)
+                    return hpo.hp.uniform(par, domain.min, domain.max)
+                elif isinstance(sampler, Normal):
+                    if quantize:
+                        return hpo.hp.qnormal(par, sampler.mean, sampler.sd,
+                                              quantize)
+                    return hpo.hp.normal(par, sampler.mean, sampler.sd)
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, Uniform):
+                    if quantize:
+                        logger.warning(
+                            "HyperOpt does not support quantization for "
+                            "integer values. Dropped quantization.")
+                    if domain.min != 0:
+                        logger.warning(
+                            "HyperOpt only allows integer sampling with "
+                            "lower bound 0. Dropped the lower bound {}".format(
+                                domain.min))
+                    if domain.max < 1:
+                        raise ValueError(
+                            "HyperOpt does not support integer sampling "
+                            "of values lower than 0. Set your maximum range "
+                            "to something above 0 (currently {})".format(
+                                domain.max))
+                    return hpo.hp.randint(par, domain.max)
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return hpo.hp.choice(par, domain.categories)
+
+            raise ValueError("HyperOpt does not support parameters of type "
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
+
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            value = resolve_value(par, domain)
+            assign_value(spec, path, value)
+
+        return spec
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 1c03dbf7309b..816fcf0cfa40 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -93,6 +93,43 @@ def testQuantized(self):
             factor = sample / 5e-4
             self.assertAlmostEqual(factor, round(factor), places=10)
 
+    def testConvertHyperOpt(self):
+        from ray.tune.suggest.hyperopt import HyperOptSearch
+        from hyperopt import hp
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = HyperOptSearch.convert_search_space(config)
+        hyperopt_config = {
+            "a": hp.choice("a", [2, 3, 4]),
+            "b": {
+                "x": hp.randint("x", 5),
+                "y": 4,
+                "z": hp.loguniform("z", np.log(1e-4), np.log(1e-2))
+            }
+        }
+
+        searcher1 = HyperOptSearch(
+            space=converted_config, random_state_seed=1234)
+        searcher2 = HyperOptSearch(
+            space=hyperopt_config, random_state_seed=1234)
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertEqual(config1["b"]["y"], 4)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
         from optuna.samplers import RandomSampler
@@ -124,6 +161,11 @@ def testConvertOptuna(self):
         config2 = searcher2.suggest("0")
 
         self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertEqual(config1["b"]["y"], 4)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
 
 
 if __name__ == "__main__":

From cba10d890739d5dddcc1e5b9a2c1aec36c28079b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 18:06:23 +0100
Subject: [PATCH 07/48] Convert search spaces to AxSearch

---
 python/ray/tune/suggest/ax.py                | 88 +++++++++++++++++++-
 python/ray/tune/suggest/hyperopt.py          |  2 +-
 python/ray/tune/suggest/optuna.py            |  2 +-
 python/ray/tune/suggest/variant_generator.py | 43 +++++++---
 python/ray/tune/tests/test_sample.py         | 57 +++++++++++++
 python/ray/tune/utils/util.py                | 12 +++
 6 files changed, 189 insertions(+), 15 deletions(-)

diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 9e58c4cb1f11..74ed7e659b58 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -1,3 +1,11 @@
+from typing import Dict
+
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
+    Quantized, Uniform
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import unflatten_dict
+
 try:
     import ax
 except ImportError:
@@ -94,7 +102,7 @@ def suggest(self, trial_id):
                 return None
         parameters, trial_index = self._ax.get_next_trial()
         self._live_trial_mapping[trial_id] = trial_index
-        return parameters
+        return unflatten_dict(parameters)
 
     def on_trial_complete(self, trial_id, result=None, error=False):
         """Notification for the completion of trial.
@@ -117,3 +125,81 @@ def _process_result(self, trial_id, result):
         metric_dict.update({on: (result[on], 0.0) for on in outcome_names})
         self._ax.complete_trial(
             trial_index=ax_trial_index, raw_data=metric_dict)
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an Ax search space.")
+
+        values = []
+
+        for path, val in resolved_vars:
+            values.append({
+                "name": "/".join(path),
+                "type": "fixed",
+                "value": val
+            })
+
+        def resolve_value(par, domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning("Ax search does not support quantization. "
+                               "Dropped quantization.")
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "float",
+                        "log_scale": True
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "float",
+                        "log_scale": False
+                    }
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "int",
+                        "log_scale": True
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "int",
+                        "log_scale": False
+                    }
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "choice",
+                        "values": domain.categories
+                    }
+
+            raise ValueError("Ax search does not support parameters of type "
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
+
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            values.append(resolve_value(par, domain))
+        return values
diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 7c1eacf8a237..83e76762f71e 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -247,7 +247,7 @@ def restore(self, checkpoint_path):
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = copy.deepcopy(spec)
-        domain_vars, grid_vars = parse_spec_vars(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if not domain_vars and not grid_vars:
             return []
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index cde4bb6a77a1..34d19292dec8 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -165,7 +165,7 @@ def restore(self, checkpoint_path):
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
-        domain_vars, grid_vars = parse_spec_vars(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if not domain_vars and not grid_vars:
             return []
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index 00040ce0b599..181296faec03 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -116,9 +116,13 @@ def _clean_value(value):
 
 
 def parse_spec_vars(spec):
-    unresolved = _unresolved_values(spec)
+    resolved, unresolved = _split_resolved_unresolved_values(spec)
+    resolved_vars = []
+    for path, value in resolved.items():
+        resolved_vars.append((path, value))
+
     if not unresolved:
-        return [], []
+        return resolved_vars, [], []
 
     grid_vars = []
     domain_vars = []
@@ -129,12 +133,12 @@ def parse_spec_vars(spec):
             domain_vars.append((path, value))
     grid_vars.sort()
 
-    return domain_vars, grid_vars
+    return resolved_vars, domain_vars, grid_vars
 
 
 def _generate_variants(spec):
     spec = copy.deepcopy(spec)
-    domain_vars, grid_vars = parse_spec_vars(spec)
+    _, domain_vars, grid_vars = parse_spec_vars(spec)
 
     if not domain_vars and not grid_vars:
         yield {}, spec
@@ -245,22 +249,37 @@ def _try_resolve(v):
     return True, v
 
 
-def _unresolved_values(spec):
-    found = {}
+def _split_resolved_unresolved_values(spec):
+    resolved_vars = {}
+    unresolved_vars = {}
     for k, v in spec.items():
         resolved, v = _try_resolve(v)
         if not resolved:
-            found[(k, )] = v
+            unresolved_vars[(k, )] = v
         elif isinstance(v, dict):
             # Recurse into a dict
-            for (path, value) in _unresolved_values(v).items():
-                found[(k, ) + path] = value
+            _resolved_children, _unresolved_children = \
+                _split_resolved_unresolved_values(v)
+            for (path, value) in _resolved_children.items():
+                resolved_vars[(k, ) + path] = value
+            for (path, value) in _unresolved_children.items():
+                unresolved_vars[(k, ) + path] = value
         elif isinstance(v, list):
             # Recurse into a list
             for i, elem in enumerate(v):
-                for (path, value) in _unresolved_values({i: elem}).items():
-                    found[(k, ) + path] = value
-    return found
+                _resolved_children, _unresolved_children = \
+                    _split_resolved_unresolved_values({i: elem})
+                for (path, value) in _resolved_children.items():
+                    resolved_vars[(k, ) + path] = value
+                for (path, value) in _unresolved_children.items():
+                    unresolved_vars[(k, ) + path] = value
+        else:
+            resolved_vars[(k, )] = v
+    return resolved_vars, unresolved_vars
+
+
+def _unresolved_values(spec):
+    return _split_resolved_unresolved_values(spec)[1]
 
 
 class _UnresolvedAccessGuard(dict):
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 816fcf0cfa40..8ff772e1fad9 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -93,6 +93,63 @@ def testQuantized(self):
             factor = sample / 5e-4
             self.assertAlmostEqual(factor, round(factor), places=10)
 
+    def testConvertAx(self):
+        from ray.tune.suggest.ax import AxSearch
+        from ax.service.ax_client import AxClient
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = AxSearch.convert_search_space(config)
+        ax_config = [
+            {
+                "name": "a",
+                "type": "choice",
+                "values": [2, 3, 4]
+            },
+            {
+                "name": "b/x",
+                "type": "range",
+                "bounds": [0, 5],
+                "value_type": "int"
+            },
+            {
+                "name": "b/y",
+                "type": "fixed",
+                "value": 4
+            },
+            {
+                "name": "b/z",
+                "type": "range",
+                "bounds": [1e-4, 1e-2],
+                "value_type": "float",
+                "log_scale": True
+            },
+        ]
+
+        client1 = AxClient(random_seed=1234)
+        client1.create_experiment(parameters=converted_config)
+        searcher1 = AxSearch(client1)
+
+        client2 = AxClient(random_seed=1234)
+        client2.create_experiment(parameters=ax_config)
+        searcher2 = AxSearch(client2)
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertEqual(config1["b"]["y"], 4)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
         from hyperopt import hp
diff --git a/python/ray/tune/utils/util.py b/python/ray/tune/utils/util.py
index fefcf7665c81..29a56f06b617 100644
--- a/python/ray/tune/utils/util.py
+++ b/python/ray/tune/utils/util.py
@@ -231,6 +231,18 @@ def flatten_dict(dt, delimiter="/"):
     return dt
 
 
+def unflatten_dict(dt, delimiter="/"):
+    """Unflatten dict. Does not support unflattening lists."""
+    out = defaultdict(dict)
+    for key, val in dt.items():
+        path = key.split(delimiter)
+        item = out
+        for k in path[:-1]:
+            item = item[k]
+        item[path[-1]] = val
+    return dict(out)
+
+
 def unflattened_lookup(flat_key, lookup, delimiter="/", **kwargs):
     """
     Unflatten `flat_key` and iteratively look up in `lookup`. E.g.

From f83ed33dae69963c608532ae3715eb32268047b0 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 19:34:17 +0100
Subject: [PATCH 08/48] Convert search spaces to BayesOpt

---
 python/ray/tune/sample.py            | 10 ++++--
 python/ray/tune/suggest/bayesopt.py  | 48 +++++++++++++++++++++++++++-
 python/ray/tune/tests/test_sample.py | 35 ++++++++++++++++++++
 3 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 65459f575b7b..04195258a77f 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -48,7 +48,7 @@ def __init__(self, min=float("-inf"), max=float("inf")):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
         return new
 
     def normal(self, mean=0., sd=1.):
@@ -90,7 +90,7 @@ def __init__(self, min, max):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
         return new
 
     def uniform(self):
@@ -267,6 +267,12 @@ def __init__(self, sampler: Sampler, q: Number):
         self.sampler = sampler
         self.q = q
 
+    def get_sampler(self):
+        sampler = self.sampler
+        if not sampler:
+            sampler = Uniform()
+        return sampler
+
     def sample(self,
                domain: Domain,
                spec: Optional[Union[List[Dict], Dict]] = None,
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index c647966fe1ae..4e4ae883e736 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -3,6 +3,12 @@
 import logging
 import pickle
 import json
+from typing import Dict
+
+from ray.tune.sample import Float, Quantized
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils.util import unflatten_dict
+
 try:  # Python 3 only -- needed for lint test.
     import bayes_opt as byo
 except ImportError:
@@ -214,7 +220,7 @@ def suggest(self, trial_id):
         self._live_trial_mapping[trial_id] = config
 
         # Return a deep copy of the mapping
-        return copy.deepcopy(config)
+        return unflatten_dict(config)
 
     def register_analysis(self, analysis):
         """Integrate the given analysis into the gaussian process.
@@ -283,3 +289,43 @@ def restore(self, checkpoint_path):
             (self.optimizer, self._buffered_trial_results,
              self._total_random_search_trials,
              self._config_counter) = pickle.load(f)
+
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an BayesOpt search space.")
+
+        if resolved_vars:
+            raise ValueError(
+                "BeaysOpt does not support fixed parameters. Please find a "
+                "different way to pass constants to your training function.")
+
+        def resolve_value(domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning("BayesOpt search does not support quantization. "
+                               "Dropped quantization.")
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if domain.sampler != None:
+                    logger.warning(
+                        "BayesOpt does not support specific sampling methods. "
+                        "The {} sampler will be dropped.".format(
+                            sampler))
+                    return (domain.min, domain.max)
+
+            raise ValueError("BayesOpt does not support parameters of type "
+                             "`{}`".format(type(domain).__name__))
+
+        bounds = {}
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            bounds[par] = resolve_value(domain)
+        return bounds
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 8ff772e1fad9..60b2dc4e5cef 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -150,6 +150,41 @@ def testConvertAx(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
+    def testConvertBayesOpt(self):
+        from ray.tune.suggest.bayesopt import BayesOptSearch
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        with self.assertRaises(ValueError):
+            converted_config = BayesOptSearch.convert_search_space(config)
+
+        config = {
+            "b": {
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        bayesopt_config = {
+            "b/z": (1e-4, 1e-2)
+        }
+
+        converted_config = BayesOptSearch.convert_search_space(config)
+
+        searcher1 = BayesOptSearch(space=converted_config, metric="none")
+        searcher2 = BayesOptSearch(space=bayesopt_config, metric="none")
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
         from hyperopt import hp

From c93fb76068154e28bbfeda04c483066b6bb5c6a2 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 26 Aug 2020 17:37:05 +0100
Subject: [PATCH 09/48] Added basic functionality and tests

---
 python/ray/tune/sample.py            | 205 +++++++++++++++++++++++++++
 python/ray/tune/tests/test_sample.py |  92 ++++++++++++
 python/requirements_tune.txt         |   1 +
 python/setup.py                      |   3 +-
 4 files changed, 299 insertions(+), 2 deletions(-)
 create mode 100644 python/ray/tune/tests/test_sample.py

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 4cac8572b417..3ba1bd6cfc83 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -1,11 +1,216 @@
 import logging
 import random
+from copy import copy
+from typing import Callable, Dict, Iterator, List, Optional, \
+    Sequence, \
+    Union
 
 import numpy as np
+from scipy import stats
 
 logger = logging.getLogger(__name__)
 
 
+class Domain:
+    _sampler = None
+
+    def set_sampler(self, sampler):
+        if self._sampler:
+            raise ValueError("You can only choose one sampler for parameter "
+                             "domains. Existing sampler for parameter {}: "
+                             "{}. Tried to add {}".format(
+                                 self.__class__.__name__, self._sampler,
+                                 sampler))
+        self._sampler = sampler
+
+    def sample(self, spec=None, size=1):
+        sampler = self._sampler
+        if not sampler:
+            sampler = Uniform()
+        return sampler.sample(self, spec=spec, size=size)
+
+
+class Float(Domain):
+    def __init__(self, min=float("-inf"), max=float("inf")):
+        self.min = min
+        self.max = max
+
+    def normal(self, mean=0., sd=1.):
+        new = copy(self)
+        new.set_sampler(Normal(mean, sd))
+        return new
+
+    def uniform(self):
+        if not self.min > float("-inf"):
+            raise ValueError(
+                "Uniform requires a minimum bound. Make sure to set the "
+                "`min` parameter of `Float()`.")
+        if not self.max < float("inf"):
+            raise ValueError(
+                "Uniform requires a maximum bound. Make sure to set the "
+                "`max` parameter of `Float()`.")
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+    def loguniform(self, base: int = 10):
+        if not self.min > 0:
+            raise ValueError(
+                "LogUniform requires a minimum bound greater than 0. "
+                "Make sure to set the `min` parameter of `Float()` correctly.")
+        if not 0 < self.max < float("inf"):
+            raise ValueError(
+                "LogUniform requires a minimum bound greater than 0. "
+                "Make sure to set the `max` parameter of `Float()` correctly.")
+        new = copy(self)
+        new.set_sampler(LogUniform(base))
+        return new
+
+
+class Integer(Domain):
+    def __init__(self, min, max):
+        self.min = min,
+        self.max = max
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Categorical(Domain):
+    def __init__(self, categories: Sequence):
+        self.categories = list(categories)
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Iterative(Domain):
+    def __init__(self, iterator: Iterator):
+        self.iterator = iterator
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Function(Domain):
+    def __init__(self, func: Callable):
+        self.func = func
+
+    def uniform(self):
+        new = copy(self)
+        new.set_sampler(Uniform())
+        return new
+
+
+class Sampler:
+    pass
+
+
+class Uniform(Sampler):
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        if isinstance(spec, list) and spec:
+            assert len(spec) == size, \
+                "Number of passed specs must match sample size"
+        elif isinstance(spec, dict) and spec:
+            assert size == 1, \
+                "Cannot sample the same parameter more than once for one spec"
+
+        if isinstance(domain, Float):
+            assert domain.min > float("-inf"), \
+                "Uniform needs a minimum bound"
+            assert 0 < domain.max < float("inf"), \
+                "Uniform needs a maximum bound"
+            return np.random.uniform(domain.min, domain.max, size=size)
+        elif isinstance(domain, Integer):
+            return np.random.randint(domain.min, domain.max, size=size)
+        elif isinstance(domain, Categorical):
+            choices = []
+            for i in range(size):
+                choices.append(random.choice(domain.categories))
+            if len(choices) == 1:
+                return choices[0]
+            return choices
+        elif isinstance(domain, Function):
+            items = []
+            for i in range(size):
+                this_spec = spec[i] if isinstance(spec, list) else spec
+                items.append(domain.func(this_spec))
+            if len(items) == 1:
+                return items[0]
+            return items
+        elif isinstance(domain, Iterative):
+            items = []
+            for i in range(size):
+                items.append(next(domain.iterator))
+            if len(items) == 1:
+                return items[0]
+            return items
+        else:
+            raise RuntimeError(
+                "Uniform sampler does not support parameters of type {}. "
+                "Allowed types: {}".format(
+                    domain.__class__.__name__,
+                    [Float, Integer, Categorical, Function, Iterative]))
+
+
+class LogUniform(Sampler):
+    def __init__(self, base: int = 10):
+        self.base = base
+        assert self.base > 0, "Base has to be strictly greater than 0"
+
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        if isinstance(domain, Float):
+            assert domain.min > 0, \
+                "LogUniform needs a minimum bound greater than 0"
+            assert 0 < domain.max < float("inf"), \
+                "LogUniform needs a maximum bound greater than 0"
+            logmin = np.log(domain.min) / np.log(self.base)
+            logmax = np.log(domain.max) / np.log(self.base)
+
+            return self.base**(np.random.uniform(logmin, logmax, size=size))
+        else:
+            raise RuntimeError(
+                "LogUniform sampler does not support parameters of type {}. "
+                "Allowed types: {}".format(domain.__class__.__name__, [Float]))
+
+
+class Normal(Sampler):
+    def __init__(self, mean: float = 0., sd: float = 0.):
+        self.mean = mean
+        self.sd = sd
+
+        assert self.sd > 0, "SD has to be strictly greater than 0"
+
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        if isinstance(domain, Float):
+            # Use a truncated normal to avoid oversampling border values
+            dist = stats.truncnorm(
+                (domain.min - self.mean) / self.sd,
+                (domain.max - self.mean) / self.sd,
+                loc=self.mean,
+                scale=self.sd)
+            return dist.rvs(size)
+        else:
+            raise ValueError(
+                "Normal sampler does not support parameters of type {}. "
+                "Allowed types: {}".format(domain.__class__.__name__, [Float]))
+
+
 class sample_from:
     """Specify that tune should sample configuration values from this function.
 
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
new file mode 100644
index 000000000000..12a7764081fc
--- /dev/null
+++ b/python/ray/tune/tests/test_sample.py
@@ -0,0 +1,92 @@
+import numpy as np
+import unittest
+
+from ray import tune
+
+
+class SearchSpaceTest(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        pass
+
+    def testBoundedFloat(self):
+        bounded = tune.sample.Float(-4.2, 8.3)
+
+        # Don't allow to specify more than one sampler
+        with self.assertRaises(ValueError):
+            bounded.normal().uniform()
+
+        # Normal
+        samples = bounded.normal(-4, 2).sample(size=1000)
+        self.assertTrue(any(-4.2 < s < 8.3 for s in samples))
+        self.assertTrue(np.mean(samples) < -2)
+
+        # Uniform
+        samples = bounded.uniform().sample(size=1000)
+        self.assertTrue(any(-4.2 < s < 8.3 for s in samples))
+        self.assertFalse(np.mean(samples) < -2)
+
+        # Loguniform
+        with self.assertRaises(ValueError):
+            bounded.loguniform().sample(size=1000)
+
+        bounded_positive = tune.sample.Float(1e-4, 1e-1)
+        samples = bounded_positive.loguniform().sample(size=1000)
+        self.assertTrue(any(1e-4 < s < 1e-1 for s in samples))
+
+    def testUnboundedFloat(self):
+        unbounded = tune.sample.Float()
+
+        # Require min and max bounds for loguniform
+        with self.assertRaises(ValueError):
+            unbounded.loguniform()
+
+    def testBoundedInt(self):
+        bounded = tune.sample.Integer(-3, 12)
+
+        samples = bounded.uniform().sample(size=1000)
+        self.assertTrue(any(-3 <= s < 12 for s in samples))
+        self.assertFalse(np.mean(samples) < 2)
+
+    def testCategorical(self):
+        categories = [-2, -1, 0, 1, 2]
+        cat = tune.sample.Categorical(categories)
+
+        samples = cat.uniform().sample(size=1000)
+        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
+        self.assertTrue(all([c in samples for c in categories]))
+
+    def testIterative(self):
+        categories = [-2, -1, 0, 1, 2]
+
+        def test_iter():
+            for i in categories:
+                yield i
+
+        itr = tune.sample.Iterative(test_iter())
+        samples = itr.uniform().sample(size=5)
+        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
+        self.assertTrue(all([c in samples for c in categories]))
+
+        itr = tune.sample.Iterative(iter(categories))
+        samples = itr.uniform().sample(size=5)
+        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
+        self.assertTrue(all([c in samples for c in categories]))
+
+    def testFunction(self):
+        def sample(spec):
+            return np.random.uniform(-4, 4)
+
+        fnc = tune.sample.Function(sample)
+
+        samples = fnc.uniform().sample(size=1000)
+        self.assertTrue(any([-4 < s < 4 for s in samples]))
+        self.assertTrue(-2 < np.mean(samples) < 2)
+
+
+if __name__ == "__main__":
+    import pytest
+    import sys
+    sys.exit(pytest.main(["-v", __file__]))
diff --git a/python/requirements_tune.txt b/python/requirements_tune.txt
index 1123fc7ed879..b61dc62393a4 100644
--- a/python/requirements_tune.txt
+++ b/python/requirements_tune.txt
@@ -19,6 +19,7 @@ optuna
 pytest-remotedata>=0.3.1
 pytorch-lightning
 scikit-optimize
+scipy
 sigopt
 smart_open
 tensorflow_probability
diff --git a/python/setup.py b/python/setup.py
index bbe2a0be44b3..0d804209af06 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -111,7 +111,7 @@
 extras = {
     "debug": [],
     "serve": ["uvicorn", "flask", "requests"],
-    "tune": ["tabulate", "tensorboardX", "pandas"]
+    "tune": ["tabulate", "tensorboardX", "pandas", "scipy"]
 }
 
 extras["rllib"] = extras["tune"] + [
@@ -121,7 +121,6 @@
     "lz4",
     "opencv-python-headless<=4.3.0.36",
     "pyyaml",
-    "scipy",
 ]
 
 extras["streaming"] = []

From 2ffbdca78be710ba1f6d2914932cdc943305dd64 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 27 Aug 2020 11:22:04 +0100
Subject: [PATCH 10/48] Feature parity with old tune search space config

---
 python/ray/tune/__init__.py                  |   4 +-
 python/ray/tune/experiment.py                |   4 +-
 python/ray/tune/sample.py                    | 139 +++++++++++--------
 python/ray/tune/schedulers/pbt.py            |  12 +-
 python/ray/tune/suggest/variant_generator.py |  30 ++--
 python/ray/tune/tests/test_var.py            |   8 +-
 6 files changed, 109 insertions(+), 88 deletions(-)

diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py
index 43a65c430a9d..13c82b5c8389 100644
--- a/python/ray/tune/__init__.py
+++ b/python/ray/tune/__init__.py
@@ -12,13 +12,13 @@
                               save_checkpoint, checkpoint_dir)
 from ray.tune.progress_reporter import (ProgressReporter, CLIReporter,
                                         JupyterNotebookReporter)
-from ray.tune.sample import (function, sample_from, uniform, choice, randint,
+from ray.tune.sample import (sample_from, uniform, choice, randint,
                              randn, loguniform)
 
 __all__ = [
     "Trainable", "DurableTrainable", "TuneError", "grid_search",
     "register_env", "register_trainable", "run", "run_experiments", "Stopper",
-    "EarlyStopping", "Experiment", "function", "sample_from", "track",
+    "EarlyStopping", "Experiment", "sample_from", "track",
     "uniform", "choice", "randint", "randn", "loguniform",
     "ExperimentAnalysis", "Analysis", "CLIReporter", "JupyterNotebookReporter",
     "ProgressReporter", "report", "get_trial_dir", "get_trial_name",
diff --git a/python/ray/tune/experiment.py b/python/ray/tune/experiment.py
index 66410d5428ce..a312c1afeb12 100644
--- a/python/ray/tune/experiment.py
+++ b/python/ray/tune/experiment.py
@@ -8,7 +8,7 @@
 from ray.tune.function_runner import detect_checkpoint_function
 from ray.tune.registry import register_trainable, get_trainable_cls
 from ray.tune.result import DEFAULT_RESULTS_DIR
-from ray.tune.sample import sample_from
+from ray.tune.sample import Domain
 from ray.tune.stopper import FunctionStopper, Stopper
 
 logger = logging.getLogger(__name__)
@@ -235,7 +235,7 @@ def register_if_needed(cls, run_object):
 
         if isinstance(run_object, str):
             return run_object
-        elif isinstance(run_object, sample_from):
+        elif isinstance(run_object, Domain):
             logger.warning("Not registering trainable. Resolving as variant.")
             return run_object
         elif isinstance(run_object, type) or callable(run_object):
diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 3ba1bd6cfc83..e5ffe1ed2694 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -29,6 +29,12 @@ def sample(self, spec=None, size=1):
             sampler = Uniform()
         return sampler.sample(self, spec=spec, size=size)
 
+    def is_grid(self):
+        return isinstance(self._sampler, Grid)
+
+    def is_function(self):
+        return False
+
 
 class Float(Domain):
     def __init__(self, min=float("-inf"), max=float("inf")):
@@ -87,6 +93,17 @@ def uniform(self):
         new.set_sampler(Uniform())
         return new
 
+    def grid(self):
+        new = copy(self)
+        new.set_sampler(Grid())
+        return new
+
+    def __len__(self):
+        return len(self.categories)
+
+    def __getitem__(self, item):
+        return self.categories[item]
+
 
 class Iterative(Domain):
     def __init__(self, iterator: Iterator):
@@ -107,6 +124,9 @@ def uniform(self):
         new.set_sampler(Uniform())
         return new
 
+    def is_function(self):
+        return True
+
 
 class Sampler:
     pass
@@ -129,9 +149,15 @@ def sample(self,
                 "Uniform needs a minimum bound"
             assert 0 < domain.max < float("inf"), \
                 "Uniform needs a maximum bound"
-            return np.random.uniform(domain.min, domain.max, size=size)
+            items = np.random.uniform(domain.min, domain.max, size=size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         elif isinstance(domain, Integer):
-            return np.random.randint(domain.min, domain.max, size=size)
+            items = np.random.randint(domain.min, domain.max, size=size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         elif isinstance(domain, Categorical):
             choices = []
             for i in range(size):
@@ -179,7 +205,10 @@ def sample(self,
             logmin = np.log(domain.min) / np.log(self.base)
             logmax = np.log(domain.max) / np.log(self.base)
 
-            return self.base**(np.random.uniform(logmin, logmax, size=size))
+            items = self.base**(np.random.uniform(logmin, logmax, size=size))
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         else:
             raise RuntimeError(
                 "LogUniform sampler does not support parameters of type {}. "
@@ -204,99 +233,91 @@ def sample(self,
                 (domain.max - self.mean) / self.sd,
                 loc=self.mean,
                 scale=self.sd)
-            return dist.rvs(size)
+            items = dist.rvs(size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
         else:
             raise ValueError(
                 "Normal sampler does not support parameters of type {}. "
                 "Allowed types: {}".format(domain.__class__.__name__, [Float]))
 
 
-class sample_from:
+class Grid(Sampler):
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        return RuntimeError("Do not call `sample()` on grid.")
+
+
+def sample_from(func):
     """Specify that tune should sample configuration values from this function.
 
     Arguments:
         func: An callable function to draw a sample from.
     """
+    return Function(func)
 
-    def __init__(self, func):
-        self.func = func
-
-    def __str__(self):
-        return "tune.sample_from({})".format(str(self.func))
-
-    def __repr__(self):
-        return "tune.sample_from({})".format(repr(self.func))
-
-
-def function(func):
-    logger.warning(
-        "DeprecationWarning: wrapping {} with tune.function() is no "
-        "longer needed".format(func))
-    return func
 
+def uniform(min, max):
+    """Sample a float value uniformly between ``min`` and ``max``.
 
-class uniform(sample_from):
-    """Wraps tune.sample_from around ``np.random.uniform``.
-
-    ``tune.uniform(1, 10)`` is equivalent to
-    ``tune.sample_from(lambda _: np.random.uniform(1, 10))``
+    Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
+    ``np.random.uniform(1, 10))``
 
     """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: np.random.uniform(*args, **kwargs))
+    return Float(min, max).uniform()
 
 
-class loguniform(sample_from):
+def loguniform(min, max, base=10):
     """Sugar for sampling in different orders of magnitude.
 
     Args:
-        min_bound (float): Lower boundary of the output interval (1e-4)
-        max_bound (float): Upper boundary of the output interval (1e-2)
+        min (float): Lower boundary of the output interval (e.g. 1e-4)
+        max (float): Upper boundary of the output interval (e.g. 1e-2)
         base (float): Base of the log. Defaults to 10.
-    """
-
-    def __init__(self, min_bound, max_bound, base=10):
-        logmin = np.log(min_bound) / np.log(base)
-        logmax = np.log(max_bound) / np.log(base)
 
-        def apply_log(_):
-            return base**(np.random.uniform(logmin, logmax))
-
-        super().__init__(apply_log)
+    """
+    return Float(min, max).loguniform(base)
 
 
-class choice(sample_from):
-    """Wraps tune.sample_from around ``random.choice``.
+def choice(categories):
+    """Sample a categorical value.
 
-    ``tune.choice([1, 2])`` is equivalent to
-    ``tune.sample_from(lambda _: random.choice([1, 2]))``
+    Sampling from ``tune.choice([1, 2])`` is equivalent to sampling from
+    ``random.choice([1, 2])``
 
     """
+    return Categorical(categories).uniform()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: random.choice(*args, **kwargs))
 
+def randint(min, max):
+    """Sample an integer value uniformly between ``min`` and ``max``.
 
-class randint(sample_from):
-    """Wraps tune.sample_from around ``np.random.randint``.
+    ``min`` is inclusive, ``max`` is exlcusive.
 
-    ``tune.randint(10)`` is equivalent to
-    ``tune.sample_from(lambda _: np.random.randint(10))``
+    Sampling from ``tune.randint(10)`` is equivalent to sampling from
+    ``np.random.randint(10)``
 
     """
+    return Integer(min, max).uniform()
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: np.random.randint(*args, **kwargs))
 
+def randn(mean: float = 0.,
+          sd: float = 1.,
+          min: float = float("-inf"),
+          max: float = float("inf")):
+    """Sample a float value normally with ``mean`` and ``sd``.
 
-class randn(sample_from):
-    """Wraps tune.sample_from around ``np.random.randn``.
+    Will truncate the normal distribution at ``min`` and ``max`` to avoid
+    oversampling the border regions.
 
-    ``tune.randn(10)`` is equivalent to
-    ``tune.sample_from(lambda _: np.random.randn(10))``
+    Args:
+        mean (float): Mean of the normal distribution. Defaults to 0.
+        sd (float): SD of the normal distribution. Defaults to 1.
+        min (float): Minimum bound. Defaults to -inf.
+        max (float): Maximum bound. Defaults to inf.
 
     """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(lambda _: np.random.randn(*args, **kwargs))
+    return Float(min, max).normal(mean, sd)
diff --git a/python/ray/tune/schedulers/pbt.py b/python/ray/tune/schedulers/pbt.py
index fd19ff9336c0..4b84b5d63b4c 100644
--- a/python/ray/tune/schedulers/pbt.py
+++ b/python/ray/tune/schedulers/pbt.py
@@ -9,7 +9,7 @@
 from ray.tune.error import TuneError
 from ray.tune.result import TRAINING_ITERATION
 from ray.tune.logger import _SafeFallbackEncoder
-from ray.tune.sample import sample_from
+from ray.tune.sample import Domain, sample_from
 from ray.tune.schedulers import FIFOScheduler, TrialScheduler
 from ray.tune.suggest.variant_generator import format_vars
 from ray.tune.trial import Trial, Checkpoint
@@ -68,8 +68,8 @@ def explore(config, mutations, resample_probability, custom_explore_fn):
                     distribution.index(config[key]) + 1)]
         else:
             if random.random() < resample_probability:
-                new_config[key] = distribution.func(None) if isinstance(
-                    distribution, sample_from) else distribution()
+                new_config[key] = distribution.sample(None) if isinstance(
+                    distribution, Domain) else distribution()
             elif random.random() > 0.5:
                 new_config[key] = config[key] * 1.2
             else:
@@ -96,8 +96,8 @@ def fill_config(config, attr, search_space):
     """Add attr to config by sampling from search_space."""
     if callable(search_space):
         config[attr] = search_space()
-    elif isinstance(search_space, sample_from):
-        config[attr] = search_space.func(None)
+    elif isinstance(search_space, Domain):
+        config[attr] = search_space.sample(None)
     elif isinstance(search_space, list):
         config[attr] = random.choice(search_space)
     elif isinstance(search_space, dict):
@@ -228,7 +228,7 @@ def __init__(self,
                  synch=False):
         for value in hyperparam_mutations.values():
             if not (isinstance(value,
-                               (list, dict, sample_from)) or callable(value)):
+                               (list, dict, Domain)) or callable(value)):
                 raise TypeError("`hyperparam_mutation` values must be either "
                                 "a List, Dict, a tune search space object, or "
                                 "callable.")
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index e772ffb4933c..4d908104eefa 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -4,7 +4,7 @@
 import random
 
 from ray.tune import TuneError
-from ray.tune.sample import sample_from
+from ray.tune.sample import Categorical, Domain, Function
 
 logger = logging.getLogger(__name__)
 
@@ -123,17 +123,17 @@ def _generate_variants(spec):
         return
 
     grid_vars = []
-    lambda_vars = []
+    domain_vars = []
     for path, value in unresolved.items():
-        if callable(value):
-            lambda_vars.append((path, value))
-        else:
+        if value.is_grid():
             grid_vars.append((path, value))
+        else:
+            domain_vars.append((path, value))
     grid_vars.sort()
 
     grid_search = _grid_search_generator(spec, grid_vars)
     for resolved_spec in grid_search:
-        resolved_vars = _resolve_lambda_vars(resolved_spec, lambda_vars)
+        resolved_vars = _resolve_domain_vars(resolved_spec, domain_vars)
         for resolved, spec in _generate_variants(resolved_spec):
             for path, value in grid_vars:
                 resolved_vars[path] = _get_value(spec, path)
@@ -160,16 +160,16 @@ def _get_value(spec, path):
     return spec
 
 
-def _resolve_lambda_vars(spec, lambda_vars):
+def _resolve_domain_vars(spec, domain_vars):
     resolved = {}
     error = True
     num_passes = 0
     while error and num_passes < _MAX_RESOLUTION_PASSES:
         num_passes += 1
         error = False
-        for path, fn in lambda_vars:
+        for path, fn in domain_vars:
             try:
-                value = fn(_UnresolvedAccessGuard(spec))
+                value = fn.sample(_UnresolvedAccessGuard(spec))
             except RecursiveDependencyError as e:
                 error = e
             except Exception:
@@ -217,13 +217,13 @@ def _is_resolved(v):
 
 
 def _try_resolve(v):
-    if isinstance(v, sample_from):
-        # Function to sample from
-        return False, v.func
+    if isinstance(v, Domain):
+        # Domain to sample from
+        return False, v
     elif isinstance(v, dict) and len(v) == 1 and "eval" in v:
         # Lambda function in eval syntax
-        return False, lambda spec: eval(
-            v["eval"], _STANDARD_IMPORTS, {"spec": spec})
+        return False, Function(
+            lambda spec: eval(v["eval"], _STANDARD_IMPORTS, {"spec": spec}))
     elif isinstance(v, dict) and len(v) == 1 and "grid_search" in v:
         # Grid search values
         grid_values = v["grid_search"]
@@ -231,7 +231,7 @@ def _try_resolve(v):
             raise TuneError(
                 "Grid search expected list of values, got: {}".format(
                     grid_values))
-        return False, grid_values
+        return False, Categorical(grid_values).grid()
     return True, v
 
 
diff --git a/python/ray/tune/tests/test_var.py b/python/ray/tune/tests/test_var.py
index 3df8e7f16edb..082a9fe0345c 100644
--- a/python/ray/tune/tests/test_var.py
+++ b/python/ray/tune/tests/test_var.py
@@ -263,13 +263,13 @@ def testNestedValues(self):
         })
 
     def testLogUniform(self):
-        sampler = tune.loguniform(1e-10, 1e-1).func
-        results = [sampler(None) for i in range(1000)]
+        sampler = tune.loguniform(1e-10, 1e-1)
+        results = sampler.sample(None, 1000)
         assert abs(np.log(min(results)) / np.log(10) - -10) < 0.1
         assert abs(np.log(max(results)) / np.log(10) - -1) < 0.1
 
-        sampler_e = tune.loguniform(np.e**-4, np.e, base=np.e).func
-        results_e = [sampler_e(None) for i in range(1000)]
+        sampler_e = tune.loguniform(np.e**-4, np.e, base=np.e)
+        results_e = sampler_e.sample(None, 1000)
         assert abs(np.log(min(results_e)) - -4) < 0.1
         assert abs(np.log(max(results_e)) - 1) < 0.1
 

From 13a153c105c5bf6537d90fa8e61d4e34026656dd Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 27 Aug 2020 18:27:21 +0100
Subject: [PATCH 11/48] Convert Optuna search spaces

---
 python/ray/tune/__init__.py                  | 14 +++----
 python/ray/tune/sample.py                    | 20 +++++----
 python/ray/tune/suggest/optuna.py            | 43 ++++++++++++++++++++
 python/ray/tune/suggest/variant_generator.py | 30 +++++++++-----
 python/ray/tune/tests/test_sample.py         | 30 ++++++++++++++
 5 files changed, 113 insertions(+), 24 deletions(-)

diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py
index 13c82b5c8389..6312fe8986dc 100644
--- a/python/ray/tune/__init__.py
+++ b/python/ray/tune/__init__.py
@@ -12,15 +12,15 @@
                               save_checkpoint, checkpoint_dir)
 from ray.tune.progress_reporter import (ProgressReporter, CLIReporter,
                                         JupyterNotebookReporter)
-from ray.tune.sample import (sample_from, uniform, choice, randint,
-                             randn, loguniform)
+from ray.tune.sample import (sample_from, uniform, choice, randint, randn,
+                             loguniform)
 
 __all__ = [
     "Trainable", "DurableTrainable", "TuneError", "grid_search",
     "register_env", "register_trainable", "run", "run_experiments", "Stopper",
-    "EarlyStopping", "Experiment", "sample_from", "track",
-    "uniform", "choice", "randint", "randn", "loguniform",
-    "ExperimentAnalysis", "Analysis", "CLIReporter", "JupyterNotebookReporter",
-    "ProgressReporter", "report", "get_trial_dir", "get_trial_name",
-    "get_trial_id", "make_checkpoint_dir", "save_checkpoint", "checkpoint_dir"
+    "EarlyStopping", "Experiment", "sample_from", "track", "uniform", "choice",
+    "randint", "randn", "loguniform", "ExperimentAnalysis", "Analysis",
+    "CLIReporter", "JupyterNotebookReporter", "ProgressReporter", "report",
+    "get_trial_dir", "get_trial_name", "get_trial_id", "make_checkpoint_dir",
+    "save_checkpoint", "checkpoint_dir"
 ]
diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index e5ffe1ed2694..03f0854be9fa 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -12,25 +12,31 @@
 
 
 class Domain:
-    _sampler = None
+    sampler = None
 
     def set_sampler(self, sampler):
-        if self._sampler:
+        if self.sampler:
             raise ValueError("You can only choose one sampler for parameter "
                              "domains. Existing sampler for parameter {}: "
                              "{}. Tried to add {}".format(
-                                 self.__class__.__name__, self._sampler,
+                                 self.__class__.__name__, self.sampler,
                                  sampler))
-        self._sampler = sampler
+        self.sampler = sampler
+
+    def has_sampler(self, cls):
+        sampler = self.sampler
+        if not sampler:
+            sampler = Uniform()
+        return isinstance(sampler, cls)
 
     def sample(self, spec=None, size=1):
-        sampler = self._sampler
+        sampler = self.sampler
         if not sampler:
             sampler = Uniform()
         return sampler.sample(self, spec=spec, size=size)
 
     def is_grid(self):
-        return isinstance(self._sampler, Grid)
+        return isinstance(self.sampler, Grid)
 
     def is_function(self):
         return False
@@ -75,7 +81,7 @@ def loguniform(self, base: int = 10):
 
 class Integer(Domain):
     def __init__(self, min, max):
-        self.min = min,
+        self.min = min
         self.max = max
 
     def uniform(self):
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 51d3efc14ee7..0a81822a9ebc 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -1,7 +1,11 @@
 import logging
 import pickle
+from typing import Dict
 
 from ray.tune.result import TRAINING_ITERATION
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, Uniform
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
 
 try:
     import optuna as ot
@@ -147,3 +151,42 @@ def restore(self, checkpoint_path):
             save_object = pickle.load(inputFile)
         self._storage, self._pruner, self._sampler, \
             self._ot_trials, self._ot_study = save_object
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec)
+        domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return []
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an Optuna search space.")
+
+        values = []
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            if isinstance(domain, Float) and domain.has_sampler(LogUniform):
+                value = param.suggest_loguniform(par, domain.min, domain.max)
+            elif isinstance(domain, Float) and domain.has_sampler(Uniform):
+                value = param.suggest_uniform(par, domain.min, domain.max)
+            elif isinstance(domain,
+                            Integer) and domain.has_sampler(LogUniform):
+                value = param.suggest_int(
+                    par, domain.min, domain.max, log=True)
+            elif isinstance(domain, Integer) and domain.has_sampler(Uniform):
+                value = param.suggest_int(
+                    par, domain.min, domain.max, log=False)
+            elif isinstance(domain,
+                            Categorical) and domain.has_sampler(Uniform):
+                value = param.suggest_categorical(par, domain.categories)
+            else:
+                raise ValueError(
+                    "Optuna search does not support parameters of type "
+                    "{} with samplers of type {}".format(
+                        type(domain), type(domain.sampler)))
+            values.append(value)
+
+        return values
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index 4d908104eefa..00040ce0b599 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -115,12 +115,10 @@ def _clean_value(value):
         return str(value).replace("/", "_")
 
 
-def _generate_variants(spec):
-    spec = copy.deepcopy(spec)
+def parse_spec_vars(spec):
     unresolved = _unresolved_values(spec)
     if not unresolved:
-        yield {}, spec
-        return
+        return [], []
 
     grid_vars = []
     domain_vars = []
@@ -131,6 +129,17 @@ def _generate_variants(spec):
             domain_vars.append((path, value))
     grid_vars.sort()
 
+    return domain_vars, grid_vars
+
+
+def _generate_variants(spec):
+    spec = copy.deepcopy(spec)
+    domain_vars, grid_vars = parse_spec_vars(spec)
+
+    if not domain_vars and not grid_vars:
+        yield {}, spec
+        return
+
     grid_search = _grid_search_generator(spec, grid_vars)
     for resolved_spec in grid_search:
         resolved_vars = _resolve_domain_vars(resolved_spec, domain_vars)
@@ -148,7 +157,7 @@ def _generate_variants(spec):
             yield resolved_vars, spec
 
 
-def _assign_value(spec, path, value):
+def assign_value(spec, path, value):
     for k in path[:-1]:
         spec = spec[k]
     spec[path[-1]] = value
@@ -167,16 +176,17 @@ def _resolve_domain_vars(spec, domain_vars):
     while error and num_passes < _MAX_RESOLUTION_PASSES:
         num_passes += 1
         error = False
-        for path, fn in domain_vars:
+        for path, domain in domain_vars:
             try:
-                value = fn.sample(_UnresolvedAccessGuard(spec))
+                value = domain.sample(_UnresolvedAccessGuard(spec))
             except RecursiveDependencyError as e:
                 error = e
             except Exception:
                 raise ValueError(
-                    "Failed to evaluate expression: {}: {}".format(path, fn))
+                    "Failed to evaluate expression: {}: {}".format(
+                        path, domain))
             else:
-                _assign_value(spec, path, value)
+                assign_value(spec, path, value)
                 resolved[path] = value
     if error:
         raise error
@@ -203,7 +213,7 @@ def increment(i):
     while value_indices[-1] < len(grid_vars[-1][1]):
         spec = copy.deepcopy(unresolved_spec)
         for i, (path, values) in enumerate(grid_vars):
-            _assign_value(spec, path, values[value_indices[i]])
+            assign_value(spec, path, values[value_indices[i]])
         yield spec
         if grid_vars:
             done = increment(0)
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 12a7764081fc..6423df6ec092 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -85,6 +85,36 @@ def sample(spec):
         self.assertTrue(any([-4 < s < 4 for s in samples]))
         self.assertTrue(-2 < np.mean(samples) < 2)
 
+    def testConvertOptuna(self):
+        from ray.tune.suggest.optuna import OptunaSearch, param
+        from optuna.samplers import RandomSampler
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = OptunaSearch.convert_search_space(config)
+        optuna_config = [
+            param.suggest_categorical("a", [2, 3, 4]),
+            param.suggest_int("b/x", 0, 5),
+            param.suggest_loguniform("b/z", 1e-4, 1e-2)
+        ]
+
+        sampler1 = RandomSampler(seed=1234)
+        searcher1 = OptunaSearch(space=converted_config, sampler=sampler1)
+
+        sampler2 = RandomSampler(seed=1234)
+        searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2)
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+
 
 if __name__ == "__main__":
     import pytest

From 6abffaa5770bddb364a9fd27fdb051ebc050d1e2 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 27 Aug 2020 18:58:42 +0100
Subject: [PATCH 12/48] Introduced quantized values

---
 python/ray/tune/sample.py            | 91 ++++++++++++++++++++++++++--
 python/ray/tune/tests/test_sample.py |  8 +++
 2 files changed, 95 insertions(+), 4 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 03f0854be9fa..f8f0242dd9e6 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -1,6 +1,7 @@
 import logging
 import random
 from copy import copy
+from numbers import Number
 from typing import Callable, Dict, Iterator, List, Optional, \
     Sequence, \
     Union
@@ -14,8 +15,8 @@
 class Domain:
     sampler = None
 
-    def set_sampler(self, sampler):
-        if self.sampler:
+    def set_sampler(self, sampler, allow_override=False):
+        if self.sampler and not allow_override:
             raise ValueError("You can only choose one sampler for parameter "
                              "domains. Existing sampler for parameter {}: "
                              "{}. Tried to add {}".format(
@@ -47,6 +48,11 @@ def __init__(self, min=float("-inf"), max=float("inf")):
         self.min = min
         self.max = max
 
+    def quantized(self, q: Number):
+        new = copy(self)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        return new
+
     def normal(self, mean=0., sd=1.):
         new = copy(self)
         new.set_sampler(Normal(mean, sd))
@@ -84,6 +90,11 @@ def __init__(self, min, max):
         self.min = min
         self.max = max
 
+    def quantized(self, q: Number):
+        new = copy(self)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        return new
+
     def uniform(self):
         new = copy(self)
         new.set_sampler(Uniform())
@@ -135,7 +146,11 @@ def is_function(self):
 
 
 class Sampler:
-    pass
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        raise NotImplementedError
 
 
 class Uniform(Sampler):
@@ -249,6 +264,22 @@ def sample(self,
                 "Allowed types: {}".format(domain.__class__.__name__, [Float]))
 
 
+class Quantized(Sampler):
+    def __init__(self, sampler: Sampler, q: Number):
+        self.sampler = sampler
+        self.q = q
+
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        values = self.sampler.sample(domain, spec, size)
+        quantized = np.round(np.divide(values, self.q)) * self.q
+        if len(quantized) == 1:
+            return quantized[0]
+        return list(quantized)
+
+
 class Grid(Sampler):
     def sample(self,
                domain: Domain,
@@ -276,18 +307,46 @@ def uniform(min, max):
     return Float(min, max).uniform()
 
 
+def quniform(min, max, q):
+    """Sample a quantized float value uniformly between ``min`` and ``max``.
+
+    Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
+    ``np.random.uniform(1, 10))``
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    """
+    return Float(min, max).uniform().quantized(q)
+
+
 def loguniform(min, max, base=10):
     """Sugar for sampling in different orders of magnitude.
 
     Args:
         min (float): Lower boundary of the output interval (e.g. 1e-4)
         max (float): Upper boundary of the output interval (e.g. 1e-2)
-        base (float): Base of the log. Defaults to 10.
+        base (int): Base of the log. Defaults to 10.
 
     """
     return Float(min, max).loguniform(base)
 
 
+def qloguniform(min, max, q, base=10):
+    """Sugar for sampling in different orders of magnitude.
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    Args:
+        min (float): Lower boundary of the output interval (e.g. 1e-4)
+        max (float): Upper boundary of the output interval (e.g. 1e-2)
+        q (float): Quantization number. The result will be rounded to an
+            integer increment of this value.
+        base (int): Base of the log. Defaults to 10.
+
+    """
+    return Float(min, max).loguniform(base).quantized(q)
+
+
 def choice(categories):
     """Sample a categorical value.
 
@@ -327,3 +386,27 @@ def randn(mean: float = 0.,
 
     """
     return Float(min, max).normal(mean, sd)
+
+
+def qrandn(q: float,
+           mean: float = 0.,
+           sd: float = 1.,
+           min: float = float("-inf"),
+           max: float = float("inf")):
+    """Sample a float value normally with ``mean`` and ``sd``.
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    Will truncate the normal distribution at ``min`` and ``max`` to avoid
+    oversampling the border regions.
+
+    Args:
+        q (float): Quantization number. The result will be rounded to an
+            integer increment of this value.
+        mean (float): Mean of the normal distribution. Defaults to 0.
+        sd (float): SD of the normal distribution. Defaults to 1.
+        min (float): Minimum bound. Defaults to -inf.
+        max (float): Maximum bound. Defaults to inf.
+
+    """
+    return Float(min, max).normal(mean, sd).quantized(q)
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 6423df6ec092..b2c42602587a 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -85,6 +85,14 @@ def sample(spec):
         self.assertTrue(any([-4 < s < 4 for s in samples]))
         self.assertTrue(-2 < np.mean(samples) < 2)
 
+    def testQuantized(self):
+        bounded_positive = tune.sample.Float(1e-4, 1e-1)
+        samples = bounded_positive.loguniform().quantized(5e-4).sample(size=10)
+
+        for sample in samples:
+            factor = sample / 5e-4
+            self.assertAlmostEqual(factor, round(factor), places=10)
+
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
         from optuna.samplers import RandomSampler

From 2cbd77b83f1f38adf4382f0653335b105cbd8ed5 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 16:31:33 +0100
Subject: [PATCH 13/48] Updated Optuna resolving

---
 python/ray/tune/sample.py            | 12 ++--
 python/ray/tune/suggest/optuna.py    | 82 +++++++++++++++++++---------
 python/ray/tune/tests/test_sample.py | 10 ++--
 3 files changed, 68 insertions(+), 36 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index f8f0242dd9e6..65459f575b7b 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -24,16 +24,14 @@ def set_sampler(self, sampler, allow_override=False):
                                  sampler))
         self.sampler = sampler
 
-    def has_sampler(self, cls):
+    def get_sampler(self):
         sampler = self.sampler
         if not sampler:
             sampler = Uniform()
-        return isinstance(sampler, cls)
+        return sampler
 
     def sample(self, spec=None, size=1):
-        sampler = self.sampler
-        if not sampler:
-            sampler = Uniform()
+        sampler = self.get_sampler()
         return sampler.sample(self, spec=spec, size=size)
 
     def is_grid(self):
@@ -50,7 +48,7 @@ def __init__(self, min=float("-inf"), max=float("inf")):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
         return new
 
     def normal(self, mean=0., sd=1.):
@@ -92,7 +90,7 @@ def __init__(self, min, max):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
         return new
 
     def uniform(self):
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 0a81822a9ebc..cde4bb6a77a1 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -1,10 +1,12 @@
+import copy
 import logging
 import pickle
 from typing import Dict
 
 from ray.tune.result import TRAINING_ITERATION
-from ray.tune.sample import Categorical, Float, Integer, LogUniform, Uniform
-from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
+    Quantized, Uniform
+from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars
 from ray.tune.utils import flatten_dict
 
 try:
@@ -57,6 +59,10 @@ class OptunaSearch(Searcher):
             minimizing or maximizing the metric attribute.
         sampler (optuna.samplers.BaseSampler): Optuna sampler used to
             draw hyperparameter configurations. Defaults to ``TPESampler``.
+        config (dict): Base config dict that gets overwritten by the Optuna
+            sampling and is returned to each Tune trial. This could e.g.
+            contain static variables or configurations that should be passed
+            to each trial.
 
     Example:
 
@@ -84,6 +90,7 @@ def __init__(
             metric="episode_reward_mean",
             mode="max",
             sampler=None,
+            config=None,
     ):
         assert ot is not None, (
             "Optuna must be installed! Run `pip install optuna`.")
@@ -95,6 +102,8 @@ def __init__(
 
         self._space = space
 
+        self._config = config or {}
+
         self._study_name = "optuna"  # Fixed study name for in-memory storage
         self._sampler = sampler or ot.samplers.TPESampler()
         assert isinstance(self._sampler, ot.samplers.BaseSampler), \
@@ -120,10 +129,11 @@ def suggest(self, trial_id):
             self._ot_trials[trial_id] = ot.trial.Trial(self._ot_study,
                                                        ot_trial_id)
         ot_trial = self._ot_trials[trial_id]
-        params = {}
+        params = copy.copy(self._config)
         for (fn, args, kwargs) in self._space:
             param_name = args[0] if len(args) > 0 else kwargs["name"]
-            params[param_name] = getattr(ot_trial, fn)(*args, **kwargs)
+            value = getattr(ot_trial, fn)(*args, **kwargs)  # Call Optuna trial
+            assign_value(params, param_name.split("/"), value)
         return params
 
     def on_trial_result(self, trial_id, result):
@@ -165,28 +175,50 @@ def convert_search_space(spec: Dict):
                 "Grid search parameters cannot be automatically converted "
                 "to an Optuna search space.")
 
+        def resolve_value(par, domain):
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        logger.warning(
+                            "Optuna does not support both quantization and "
+                            "sampling from LogUniform. Dropped quantization.")
+                    return param.suggest_loguniform(par, domain.min,
+                                                    domain.max)
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return param.suggest_discrete_uniform(
+                            par, domain.min, domain.max, quantize)
+                    return param.suggest_uniform(par, domain.min, domain.max)
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        logger.warning(
+                            "Optuna does not support both quantization and "
+                            "sampling from LogUniform. Dropped quantization.")
+                    return param.suggest_int(
+                        par, domain.min, domain.max, log=True)
+                elif isinstance(sampler, Uniform):
+                    return param.suggest_int(
+                        par, domain.min, domain.max, step=quantize or 1)
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return param.suggest_categorical(par, domain.categories)
+
+            raise ValueError(
+                "Optuna search does not support parameters of type "
+                "`{}` with samplers of type `{}`".format(
+                    type(domain).__name__,
+                    type(domain.sampler).__name__))
+
         values = []
         for path, domain in domain_vars:
             par = "/".join(path)
-            if isinstance(domain, Float) and domain.has_sampler(LogUniform):
-                value = param.suggest_loguniform(par, domain.min, domain.max)
-            elif isinstance(domain, Float) and domain.has_sampler(Uniform):
-                value = param.suggest_uniform(par, domain.min, domain.max)
-            elif isinstance(domain,
-                            Integer) and domain.has_sampler(LogUniform):
-                value = param.suggest_int(
-                    par, domain.min, domain.max, log=True)
-            elif isinstance(domain, Integer) and domain.has_sampler(Uniform):
-                value = param.suggest_int(
-                    par, domain.min, domain.max, log=False)
-            elif isinstance(domain,
-                            Categorical) and domain.has_sampler(Uniform):
-                value = param.suggest_categorical(par, domain.categories)
-            else:
-                raise ValueError(
-                    "Optuna search does not support parameters of type "
-                    "{} with samplers of type {}".format(
-                        type(domain), type(domain.sampler)))
-            values.append(value)
-
+            values.append(resolve_value(par, domain))
         return values
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index b2c42602587a..1c03dbf7309b 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -100,7 +100,7 @@ def testConvertOptuna(self):
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
-                "x": tune.sample.Integer(0, 5),
+                "x": tune.sample.Integer(0, 5).quantized(2),
                 "y": 4,
                 "z": tune.sample.Float(1e-4, 1e-2).loguniform()
             }
@@ -108,15 +108,17 @@ def testConvertOptuna(self):
         converted_config = OptunaSearch.convert_search_space(config)
         optuna_config = [
             param.suggest_categorical("a", [2, 3, 4]),
-            param.suggest_int("b/x", 0, 5),
+            param.suggest_int("b/x", 0, 5, 2),
             param.suggest_loguniform("b/z", 1e-4, 1e-2)
         ]
 
         sampler1 = RandomSampler(seed=1234)
-        searcher1 = OptunaSearch(space=converted_config, sampler=sampler1)
+        searcher1 = OptunaSearch(
+            space=converted_config, sampler=sampler1, config=config)
 
         sampler2 = RandomSampler(seed=1234)
-        searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2)
+        searcher2 = OptunaSearch(
+            space=optuna_config, sampler=sampler2, config=config)
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")

From c68b9c4b3cad1df7eb0bb3b7fd51ddea16851f4b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 17:06:09 +0100
Subject: [PATCH 14/48] Added HyperOpt search space conversion

---
 python/ray/tune/suggest/hyperopt.py  | 81 ++++++++++++++++++++++++++++
 python/ray/tune/tests/test_sample.py | 42 +++++++++++++++
 2 files changed, 123 insertions(+)

diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 5bdb1cbbdd08..7c1eacf8a237 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -1,8 +1,16 @@
+from typing import Dict
+
 import numpy as np
 import copy
 import logging
 from functools import partial
 import pickle
+
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, Normal, \
+    Quantized, \
+    Uniform
+from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars
+
 try:
     hyperopt_logger = logging.getLogger("hyperopt")
     hyperopt_logger.setLevel(logging.WARNING)
@@ -235,3 +243,76 @@ def restore(self, checkpoint_path):
             self.rstate.set_state(trials_object[1])
         else:
             self.set_state(trials_object)
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = copy.deepcopy(spec)
+        domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return []
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a HyperOpt search space.")
+
+        def resolve_value(par, domain):
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    if quantize:
+                        return hpo.hp.qloguniform(par, domain.min, domain.max,
+                                                  quantize)
+                    return hpo.hp.loguniform(par, np.log(domain.min),
+                                             np.log(domain.max))
+                elif isinstance(sampler, Uniform):
+                    if quantize:
+                        return hpo.hp.quniform(par, domain.min, domain.max,
+                                               quantize)
+                    return hpo.hp.uniform(par, domain.min, domain.max)
+                elif isinstance(sampler, Normal):
+                    if quantize:
+                        return hpo.hp.qnormal(par, sampler.mean, sampler.sd,
+                                              quantize)
+                    return hpo.hp.normal(par, sampler.mean, sampler.sd)
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, Uniform):
+                    if quantize:
+                        logger.warning(
+                            "HyperOpt does not support quantization for "
+                            "integer values. Dropped quantization.")
+                    if domain.min != 0:
+                        logger.warning(
+                            "HyperOpt only allows integer sampling with "
+                            "lower bound 0. Dropped the lower bound {}".format(
+                                domain.min))
+                    if domain.max < 1:
+                        raise ValueError(
+                            "HyperOpt does not support integer sampling "
+                            "of values lower than 0. Set your maximum range "
+                            "to something above 0 (currently {})".format(
+                                domain.max))
+                    return hpo.hp.randint(par, domain.max)
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return hpo.hp.choice(par, domain.categories)
+
+            raise ValueError("HyperOpt does not support parameters of type "
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
+
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            value = resolve_value(par, domain)
+            assign_value(spec, path, value)
+
+        return spec
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 1c03dbf7309b..816fcf0cfa40 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -93,6 +93,43 @@ def testQuantized(self):
             factor = sample / 5e-4
             self.assertAlmostEqual(factor, round(factor), places=10)
 
+    def testConvertHyperOpt(self):
+        from ray.tune.suggest.hyperopt import HyperOptSearch
+        from hyperopt import hp
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = HyperOptSearch.convert_search_space(config)
+        hyperopt_config = {
+            "a": hp.choice("a", [2, 3, 4]),
+            "b": {
+                "x": hp.randint("x", 5),
+                "y": 4,
+                "z": hp.loguniform("z", np.log(1e-4), np.log(1e-2))
+            }
+        }
+
+        searcher1 = HyperOptSearch(
+            space=converted_config, random_state_seed=1234)
+        searcher2 = HyperOptSearch(
+            space=hyperopt_config, random_state_seed=1234)
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertEqual(config1["b"]["y"], 4)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
         from optuna.samplers import RandomSampler
@@ -124,6 +161,11 @@ def testConvertOptuna(self):
         config2 = searcher2.suggest("0")
 
         self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertEqual(config1["b"]["y"], 4)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
 
 
 if __name__ == "__main__":

From 1167c752963f3f2433756b911ed722993937ccce Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 18:06:23 +0100
Subject: [PATCH 15/48] Convert search spaces to AxSearch

---
 python/ray/tune/suggest/ax.py                | 88 +++++++++++++++++++-
 python/ray/tune/suggest/hyperopt.py          |  2 +-
 python/ray/tune/suggest/optuna.py            |  2 +-
 python/ray/tune/suggest/variant_generator.py | 43 +++++++---
 python/ray/tune/tests/test_sample.py         | 57 +++++++++++++
 python/ray/tune/utils/util.py                | 12 +++
 6 files changed, 189 insertions(+), 15 deletions(-)

diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 9e58c4cb1f11..74ed7e659b58 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -1,3 +1,11 @@
+from typing import Dict
+
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
+    Quantized, Uniform
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import unflatten_dict
+
 try:
     import ax
 except ImportError:
@@ -94,7 +102,7 @@ def suggest(self, trial_id):
                 return None
         parameters, trial_index = self._ax.get_next_trial()
         self._live_trial_mapping[trial_id] = trial_index
-        return parameters
+        return unflatten_dict(parameters)
 
     def on_trial_complete(self, trial_id, result=None, error=False):
         """Notification for the completion of trial.
@@ -117,3 +125,81 @@ def _process_result(self, trial_id, result):
         metric_dict.update({on: (result[on], 0.0) for on in outcome_names})
         self._ax.complete_trial(
             trial_index=ax_trial_index, raw_data=metric_dict)
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an Ax search space.")
+
+        values = []
+
+        for path, val in resolved_vars:
+            values.append({
+                "name": "/".join(path),
+                "type": "fixed",
+                "value": val
+            })
+
+        def resolve_value(par, domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning("Ax search does not support quantization. "
+                               "Dropped quantization.")
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "float",
+                        "log_scale": True
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "float",
+                        "log_scale": False
+                    }
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, LogUniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "int",
+                        "log_scale": True
+                    }
+                elif isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "range",
+                        "bounds": [domain.min, domain.max],
+                        "value_type": "int",
+                        "log_scale": False
+                    }
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return {
+                        "name": par,
+                        "type": "choice",
+                        "values": domain.categories
+                    }
+
+            raise ValueError("Ax search does not support parameters of type "
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
+
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            values.append(resolve_value(par, domain))
+        return values
diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 7c1eacf8a237..83e76762f71e 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -247,7 +247,7 @@ def restore(self, checkpoint_path):
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = copy.deepcopy(spec)
-        domain_vars, grid_vars = parse_spec_vars(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if not domain_vars and not grid_vars:
             return []
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index cde4bb6a77a1..34d19292dec8 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -165,7 +165,7 @@ def restore(self, checkpoint_path):
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
-        domain_vars, grid_vars = parse_spec_vars(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if not domain_vars and not grid_vars:
             return []
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index 00040ce0b599..181296faec03 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -116,9 +116,13 @@ def _clean_value(value):
 
 
 def parse_spec_vars(spec):
-    unresolved = _unresolved_values(spec)
+    resolved, unresolved = _split_resolved_unresolved_values(spec)
+    resolved_vars = []
+    for path, value in resolved.items():
+        resolved_vars.append((path, value))
+
     if not unresolved:
-        return [], []
+        return resolved_vars, [], []
 
     grid_vars = []
     domain_vars = []
@@ -129,12 +133,12 @@ def parse_spec_vars(spec):
             domain_vars.append((path, value))
     grid_vars.sort()
 
-    return domain_vars, grid_vars
+    return resolved_vars, domain_vars, grid_vars
 
 
 def _generate_variants(spec):
     spec = copy.deepcopy(spec)
-    domain_vars, grid_vars = parse_spec_vars(spec)
+    _, domain_vars, grid_vars = parse_spec_vars(spec)
 
     if not domain_vars and not grid_vars:
         yield {}, spec
@@ -245,22 +249,37 @@ def _try_resolve(v):
     return True, v
 
 
-def _unresolved_values(spec):
-    found = {}
+def _split_resolved_unresolved_values(spec):
+    resolved_vars = {}
+    unresolved_vars = {}
     for k, v in spec.items():
         resolved, v = _try_resolve(v)
         if not resolved:
-            found[(k, )] = v
+            unresolved_vars[(k, )] = v
         elif isinstance(v, dict):
             # Recurse into a dict
-            for (path, value) in _unresolved_values(v).items():
-                found[(k, ) + path] = value
+            _resolved_children, _unresolved_children = \
+                _split_resolved_unresolved_values(v)
+            for (path, value) in _resolved_children.items():
+                resolved_vars[(k, ) + path] = value
+            for (path, value) in _unresolved_children.items():
+                unresolved_vars[(k, ) + path] = value
         elif isinstance(v, list):
             # Recurse into a list
             for i, elem in enumerate(v):
-                for (path, value) in _unresolved_values({i: elem}).items():
-                    found[(k, ) + path] = value
-    return found
+                _resolved_children, _unresolved_children = \
+                    _split_resolved_unresolved_values({i: elem})
+                for (path, value) in _resolved_children.items():
+                    resolved_vars[(k, ) + path] = value
+                for (path, value) in _unresolved_children.items():
+                    unresolved_vars[(k, ) + path] = value
+        else:
+            resolved_vars[(k, )] = v
+    return resolved_vars, unresolved_vars
+
+
+def _unresolved_values(spec):
+    return _split_resolved_unresolved_values(spec)[1]
 
 
 class _UnresolvedAccessGuard(dict):
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 816fcf0cfa40..8ff772e1fad9 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -93,6 +93,63 @@ def testQuantized(self):
             factor = sample / 5e-4
             self.assertAlmostEqual(factor, round(factor), places=10)
 
+    def testConvertAx(self):
+        from ray.tune.suggest.ax import AxSearch
+        from ax.service.ax_client import AxClient
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = AxSearch.convert_search_space(config)
+        ax_config = [
+            {
+                "name": "a",
+                "type": "choice",
+                "values": [2, 3, 4]
+            },
+            {
+                "name": "b/x",
+                "type": "range",
+                "bounds": [0, 5],
+                "value_type": "int"
+            },
+            {
+                "name": "b/y",
+                "type": "fixed",
+                "value": 4
+            },
+            {
+                "name": "b/z",
+                "type": "range",
+                "bounds": [1e-4, 1e-2],
+                "value_type": "float",
+                "log_scale": True
+            },
+        ]
+
+        client1 = AxClient(random_seed=1234)
+        client1.create_experiment(parameters=converted_config)
+        searcher1 = AxSearch(client1)
+
+        client2 = AxClient(random_seed=1234)
+        client2.create_experiment(parameters=ax_config)
+        searcher2 = AxSearch(client2)
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertEqual(config1["b"]["y"], 4)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
         from hyperopt import hp
diff --git a/python/ray/tune/utils/util.py b/python/ray/tune/utils/util.py
index 14af9d998df1..8b8136657953 100644
--- a/python/ray/tune/utils/util.py
+++ b/python/ray/tune/utils/util.py
@@ -231,6 +231,18 @@ def flatten_dict(dt, delimiter="/"):
     return dt
 
 
+def unflatten_dict(dt, delimiter="/"):
+    """Unflatten dict. Does not support unflattening lists."""
+    out = defaultdict(dict)
+    for key, val in dt.items():
+        path = key.split(delimiter)
+        item = out
+        for k in path[:-1]:
+            item = item[k]
+        item[path[-1]] = val
+    return dict(out)
+
+
 def unflattened_lookup(flat_key, lookup, delimiter="/", **kwargs):
     """
     Unflatten `flat_key` and iteratively look up in `lookup`. E.g.

From a4f2d2da0e7b81526b930a75b0904005cfe568c4 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Fri, 28 Aug 2020 19:34:17 +0100
Subject: [PATCH 16/48] Convert search spaces to BayesOpt

---
 python/ray/tune/sample.py            | 10 ++++--
 python/ray/tune/suggest/bayesopt.py  | 48 +++++++++++++++++++++++++++-
 python/ray/tune/tests/test_sample.py | 35 ++++++++++++++++++++
 3 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 65459f575b7b..04195258a77f 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -48,7 +48,7 @@ def __init__(self, min=float("-inf"), max=float("inf")):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
         return new
 
     def normal(self, mean=0., sd=1.):
@@ -90,7 +90,7 @@ def __init__(self, min, max):
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
+        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
         return new
 
     def uniform(self):
@@ -267,6 +267,12 @@ def __init__(self, sampler: Sampler, q: Number):
         self.sampler = sampler
         self.q = q
 
+    def get_sampler(self):
+        sampler = self.sampler
+        if not sampler:
+            sampler = Uniform()
+        return sampler
+
     def sample(self,
                domain: Domain,
                spec: Optional[Union[List[Dict], Dict]] = None,
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index c647966fe1ae..4e4ae883e736 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -3,6 +3,12 @@
 import logging
 import pickle
 import json
+from typing import Dict
+
+from ray.tune.sample import Float, Quantized
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils.util import unflatten_dict
+
 try:  # Python 3 only -- needed for lint test.
     import bayes_opt as byo
 except ImportError:
@@ -214,7 +220,7 @@ def suggest(self, trial_id):
         self._live_trial_mapping[trial_id] = config
 
         # Return a deep copy of the mapping
-        return copy.deepcopy(config)
+        return unflatten_dict(config)
 
     def register_analysis(self, analysis):
         """Integrate the given analysis into the gaussian process.
@@ -283,3 +289,43 @@ def restore(self, checkpoint_path):
             (self.optimizer, self._buffered_trial_results,
              self._total_random_search_trials,
              self._config_counter) = pickle.load(f)
+
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to an BayesOpt search space.")
+
+        if resolved_vars:
+            raise ValueError(
+                "BeaysOpt does not support fixed parameters. Please find a "
+                "different way to pass constants to your training function.")
+
+        def resolve_value(domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning("BayesOpt search does not support quantization. "
+                               "Dropped quantization.")
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if domain.sampler != None:
+                    logger.warning(
+                        "BayesOpt does not support specific sampling methods. "
+                        "The {} sampler will be dropped.".format(
+                            sampler))
+                    return (domain.min, domain.max)
+
+            raise ValueError("BayesOpt does not support parameters of type "
+                             "`{}`".format(type(domain).__name__))
+
+        bounds = {}
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            bounds[par] = resolve_value(domain)
+        return bounds
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 8ff772e1fad9..60b2dc4e5cef 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -150,6 +150,41 @@ def testConvertAx(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
+    def testConvertBayesOpt(self):
+        from ray.tune.suggest.bayesopt import BayesOptSearch
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        with self.assertRaises(ValueError):
+            converted_config = BayesOptSearch.convert_search_space(config)
+
+        config = {
+            "b": {
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        bayesopt_config = {
+            "b/z": (1e-4, 1e-2)
+        }
+
+        converted_config = BayesOptSearch.convert_search_space(config)
+
+        searcher1 = BayesOptSearch(space=converted_config, metric="none")
+        searcher2 = BayesOptSearch(space=bayesopt_config, metric="none")
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
         from hyperopt import hp

From bd7ed77500eb6e1b01124cb5aa021110c01ae41b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 31 Aug 2020 14:45:22 +0100
Subject: [PATCH 17/48] Re-factored samplers into domain classes

---
 python/ray/tune/sample.py            | 321 ++++++++++++++-------------
 python/ray/tune/suggest/bayesopt.py  |  16 +-
 python/ray/tune/tests/test_sample.py |  10 +-
 3 files changed, 174 insertions(+), 173 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 04195258a77f..da44ae46c406 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -2,7 +2,7 @@
 import random
 from copy import copy
 from numbers import Number
-from typing import Callable, Dict, Iterator, List, Optional, \
+from typing import Any, Callable, Dict, Iterator, List, Optional, \
     Sequence, \
     Union
 
@@ -14,6 +14,7 @@
 
 class Domain:
     sampler = None
+    default_sampler_cls = None
 
     def set_sampler(self, sampler, allow_override=False):
         if self.sampler and not allow_override:
@@ -27,7 +28,7 @@ def set_sampler(self, sampler, allow_override=False):
     def get_sampler(self):
         sampler = self.sampler
         if not sampler:
-            sampler = Uniform()
+            sampler = self.default_sampler_cls()
         return sampler
 
     def sample(self, spec=None, size=1):
@@ -41,19 +42,91 @@ def is_function(self):
         return False
 
 
+class Sampler:
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        raise NotImplementedError
+
+
+class Grid(Sampler):
+    """Dummy sampler used for grid search"""
+
+    def sample(self,
+               domain: Domain,
+               spec: Optional[Union[List[Dict], Dict]] = None,
+               size: int = 1):
+        return RuntimeError("Do not call `sample()` on grid.")
+
+
 class Float(Domain):
+    class _Uniform(Sampler):
+        def sample(self,
+                   domain: "Float",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
+            assert domain.min > float("-inf"), \
+                "Uniform needs a minimum bound"
+            assert 0 < domain.max < float("inf"), \
+                "Uniform needs a maximum bound"
+            items = np.random.uniform(domain.min, domain.max, size=size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
+
+    class _LogUniform(Sampler):
+        def __init__(self, base: int = 10):
+            self.base = base
+            assert self.base > 0, "Base has to be strictly greater than 0"
+
+        def sample(self,
+                   domain: "Float",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
+            assert domain.min > 0, \
+                "LogUniform needs a minimum bound greater than 0"
+            assert 0 < domain.max < float("inf"), \
+                "LogUniform needs a maximum bound greater than 0"
+            logmin = np.log(domain.min) / np.log(self.base)
+            logmax = np.log(domain.max) / np.log(self.base)
+
+            items = self.base**(np.random.uniform(logmin, logmax, size=size))
+            if len(items) == 1:
+                return items[0]
+            return list(items)
+
+    class _Normal(Sampler):
+        def __init__(self, mean: float = 0., sd: float = 0.):
+            self.mean = mean
+            self.sd = sd
+
+            assert self.sd > 0, "SD has to be strictly greater than 0"
+
+        def sample(self,
+                   domain: "Float",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
+            # Use a truncated normal to avoid oversampling border values
+            dist = stats.truncnorm(
+                (domain.min - self.mean) / self.sd,
+                (domain.max - self.mean) / self.sd,
+                loc=self.mean,
+                scale=self.sd)
+            items = dist.rvs(size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
+
+    default_sampler_cls = _Uniform
+
     def __init__(self, min=float("-inf"), max=float("inf")):
         self.min = min
         self.max = max
 
-    def quantized(self, q: Number):
-        new = copy(self)
-        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
-        return new
-
     def normal(self, mean=0., sd=1.):
         new = copy(self)
-        new.set_sampler(Normal(mean, sd))
+        new.set_sampler(self._Normal(mean, sd))
         return new
 
     def uniform(self):
@@ -66,7 +139,7 @@ def uniform(self):
                 "Uniform requires a maximum bound. Make sure to set the "
                 "`max` parameter of `Float()`.")
         new = copy(self)
-        new.set_sampler(Uniform())
+        new.set_sampler(self._Uniform())
         return new
 
     def loguniform(self, base: int = 10):
@@ -79,33 +152,64 @@ def loguniform(self, base: int = 10):
                 "LogUniform requires a minimum bound greater than 0. "
                 "Make sure to set the `max` parameter of `Float()` correctly.")
         new = copy(self)
-        new.set_sampler(LogUniform(base))
+        new.set_sampler(self._LogUniform(base))
+        return new
+
+    def quantized(self, q: Number):
+        new = copy(self)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
         return new
 
 
 class Integer(Domain):
+    class _Uniform(Sampler):
+        def sample(self,
+                   domain: "Integer",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
+            items = np.random.randint(domain.min, domain.max, size=size)
+            if len(items) == 1:
+                return items[0]
+            return list(items)
+
+    default_sampler_cls = _Uniform
+
     def __init__(self, min, max):
         self.min = min
         self.max = max
 
     def quantized(self, q: Number):
         new = copy(self)
-        new.set_sampler(Quantized(new.sampler, q), allow_override=True)
+        new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
         return new
 
     def uniform(self):
         new = copy(self)
-        new.set_sampler(Uniform())
+        new.set_sampler(self._Uniform())
         return new
 
 
 class Categorical(Domain):
+    class _Uniform(Sampler):
+        def sample(self,
+                   domain: "Categorical",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
+            choices = []
+            for i in range(size):
+                choices.append(random.choice(domain.categories))
+            if len(choices) == 1:
+                return choices[0]
+            return choices
+
+    default_sampler_cls = _Uniform
+
     def __init__(self, categories: Sequence):
         self.categories = list(categories)
 
     def uniform(self):
         new = copy(self)
-        new.set_sampler(Uniform())
+        new.set_sampler(self._Uniform())
         return new
 
     def grid(self):
@@ -121,145 +225,45 @@ def __getitem__(self, item):
 
 
 class Iterative(Domain):
-    def __init__(self, iterator: Iterator):
-        self.iterator = iterator
-
-    def uniform(self):
-        new = copy(self)
-        new.set_sampler(Uniform())
-        return new
-
-
-class Function(Domain):
-    def __init__(self, func: Callable):
-        self.func = func
-
-    def uniform(self):
-        new = copy(self)
-        new.set_sampler(Uniform())
-        return new
-
-    def is_function(self):
-        return True
-
-
-class Sampler:
-    def sample(self,
-               domain: Domain,
-               spec: Optional[Union[List[Dict], Dict]] = None,
-               size: int = 1):
-        raise NotImplementedError
-
-
-class Uniform(Sampler):
-    def sample(self,
-               domain: Domain,
-               spec: Optional[Union[List[Dict], Dict]] = None,
-               size: int = 1):
-        if isinstance(spec, list) and spec:
-            assert len(spec) == size, \
-                "Number of passed specs must match sample size"
-        elif isinstance(spec, dict) and spec:
-            assert size == 1, \
-                "Cannot sample the same parameter more than once for one spec"
-
-        if isinstance(domain, Float):
-            assert domain.min > float("-inf"), \
-                "Uniform needs a minimum bound"
-            assert 0 < domain.max < float("inf"), \
-                "Uniform needs a maximum bound"
-            items = np.random.uniform(domain.min, domain.max, size=size)
-            if len(items) == 1:
-                return items[0]
-            return list(items)
-        elif isinstance(domain, Integer):
-            items = np.random.randint(domain.min, domain.max, size=size)
-            if len(items) == 1:
-                return items[0]
-            return list(items)
-        elif isinstance(domain, Categorical):
-            choices = []
-            for i in range(size):
-                choices.append(random.choice(domain.categories))
-            if len(choices) == 1:
-                return choices[0]
-            return choices
-        elif isinstance(domain, Function):
-            items = []
-            for i in range(size):
-                this_spec = spec[i] if isinstance(spec, list) else spec
-                items.append(domain.func(this_spec))
-            if len(items) == 1:
-                return items[0]
-            return items
-        elif isinstance(domain, Iterative):
+    class _NextSampler(Sampler):
+        def sample(self,
+                   domain: "Iterative",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
             items = []
             for i in range(size):
                 items.append(next(domain.iterator))
             if len(items) == 1:
                 return items[0]
             return items
-        else:
-            raise RuntimeError(
-                "Uniform sampler does not support parameters of type {}. "
-                "Allowed types: {}".format(
-                    domain.__class__.__name__,
-                    [Float, Integer, Categorical, Function, Iterative]))
 
+    default_sampler_cls = _NextSampler
 
-class LogUniform(Sampler):
-    def __init__(self, base: int = 10):
-        self.base = base
-        assert self.base > 0, "Base has to be strictly greater than 0"
+    def __init__(self, iterator: Iterator):
+        self.iterator = iterator
 
-    def sample(self,
-               domain: Domain,
-               spec: Optional[Union[List[Dict], Dict]] = None,
-               size: int = 1):
-        if isinstance(domain, Float):
-            assert domain.min > 0, \
-                "LogUniform needs a minimum bound greater than 0"
-            assert 0 < domain.max < float("inf"), \
-                "LogUniform needs a maximum bound greater than 0"
-            logmin = np.log(domain.min) / np.log(self.base)
-            logmax = np.log(domain.max) / np.log(self.base)
 
-            items = self.base**(np.random.uniform(logmin, logmax, size=size))
+class Function(Domain):
+    class _CallSampler(Sampler):
+        def sample(self,
+                   domain: "Function",
+                   spec: Optional[Union[List[Dict], Dict]] = None,
+                   size: int = 1):
+            items = []
+            for i in range(size):
+                this_spec = spec[i] if isinstance(spec, list) else spec
+                items.append(domain.func(this_spec))
             if len(items) == 1:
                 return items[0]
-            return list(items)
-        else:
-            raise RuntimeError(
-                "LogUniform sampler does not support parameters of type {}. "
-                "Allowed types: {}".format(domain.__class__.__name__, [Float]))
-
+            return items
 
-class Normal(Sampler):
-    def __init__(self, mean: float = 0., sd: float = 0.):
-        self.mean = mean
-        self.sd = sd
+    default_sampler_cls = _CallSampler
 
-        assert self.sd > 0, "SD has to be strictly greater than 0"
+    def __init__(self, func: Callable):
+        self.func = func
 
-    def sample(self,
-               domain: Domain,
-               spec: Optional[Union[List[Dict], Dict]] = None,
-               size: int = 1):
-        if isinstance(domain, Float):
-            # Use a truncated normal to avoid oversampling border values
-            dist = stats.truncnorm(
-                (domain.min - self.mean) / self.sd,
-                (domain.max - self.mean) / self.sd,
-                loc=self.mean,
-                scale=self.sd)
-            items = dist.rvs(size)
-            if len(items) == 1:
-                return items[0]
-            return list(items)
-        else:
-            raise ValueError(
-                "Normal sampler does not support parameters of type {}. "
-                "Allowed types: {}".format(domain.__class__.__name__, [Float]))
+    def is_function(self):
+        return True
 
 
 class Quantized(Sampler):
@@ -267,11 +271,10 @@ def __init__(self, sampler: Sampler, q: Number):
         self.sampler = sampler
         self.q = q
 
+        assert self.sampler, "Quantized() expects a sampler instance"
+
     def get_sampler(self):
-        sampler = self.sampler
-        if not sampler:
-            sampler = Uniform()
-        return sampler
+        return self.sampler
 
     def sample(self,
                domain: Domain,
@@ -284,15 +287,7 @@ def sample(self,
         return list(quantized)
 
 
-class Grid(Sampler):
-    def sample(self,
-               domain: Domain,
-               spec: Optional[Union[List[Dict], Dict]] = None,
-               size: int = 1):
-        return RuntimeError("Do not call `sample()` on grid.")
-
-
-def sample_from(func):
+def sample_from(func: Callable[[Dict], Any]):
     """Specify that tune should sample configuration values from this function.
 
     Arguments:
@@ -301,7 +296,7 @@ def sample_from(func):
     return Function(func)
 
 
-def uniform(min, max):
+def uniform(min: float, max: float):
     """Sample a float value uniformly between ``min`` and ``max``.
 
     Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
@@ -311,7 +306,7 @@ def uniform(min, max):
     return Float(min, max).uniform()
 
 
-def quniform(min, max, q):
+def quniform(min: float, max: float, q: float):
     """Sample a quantized float value uniformly between ``min`` and ``max``.
 
     Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
@@ -323,7 +318,7 @@ def quniform(min, max, q):
     return Float(min, max).uniform().quantized(q)
 
 
-def loguniform(min, max, base=10):
+def loguniform(min: float, max: float, base: float = 10):
     """Sugar for sampling in different orders of magnitude.
 
     Args:
@@ -351,7 +346,7 @@ def qloguniform(min, max, q, base=10):
     return Float(min, max).loguniform(base).quantized(q)
 
 
-def choice(categories):
+def choice(categories: List):
     """Sample a categorical value.
 
     Sampling from ``tune.choice([1, 2])`` is equivalent to sampling from
@@ -361,10 +356,10 @@ def choice(categories):
     return Categorical(categories).uniform()
 
 
-def randint(min, max):
+def randint(min: int, max: int):
     """Sample an integer value uniformly between ``min`` and ``max``.
 
-    ``min`` is inclusive, ``max`` is exlcusive.
+    ``min`` is inclusive, ``max`` is exclusive.
 
     Sampling from ``tune.randint(10)`` is equivalent to sampling from
     ``np.random.randint(10)``
@@ -373,6 +368,20 @@ def randint(min, max):
     return Integer(min, max).uniform()
 
 
+def qrandint(min: int, max: int, q: int = 1):
+    """Sample an integer value uniformly between ``min`` and ``max``.
+
+    ``min`` is inclusive, ``max`` is exclusive.
+
+    The value will be quantized, i.e. rounded to an integer increment of ``q``.
+
+    Sampling from ``tune.randint(10)`` is equivalent to sampling from
+    ``np.random.randint(10)``
+
+    """
+    return Integer(min, max).uniform().quantized(q)
+
+
 def randn(mean: float = 0.,
           sd: float = 1.,
           min: float = float("-inf"),
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index 4e4ae883e736..d731b84af135 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -1,4 +1,3 @@
-import copy
 from collections import defaultdict
 import logging
 import pickle
@@ -290,7 +289,6 @@ def restore(self, checkpoint_path):
              self._total_random_search_trials,
              self._config_counter) = pickle.load(f)
 
-
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
@@ -299,26 +297,26 @@ def convert_search_space(spec: Dict):
         if grid_vars:
             raise ValueError(
                 "Grid search parameters cannot be automatically converted "
-                "to an BayesOpt search space.")
+                "to a BayesOpt search space.")
 
         if resolved_vars:
             raise ValueError(
-                "BeaysOpt does not support fixed parameters. Please find a "
+                "BayesOpt does not support fixed parameters. Please find a "
                 "different way to pass constants to your training function.")
 
         def resolve_value(domain):
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):
-                logger.warning("BayesOpt search does not support quantization. "
-                               "Dropped quantization.")
+                logger.warning(
+                    "BayesOpt search does not support quantization. "
+                    "Dropped quantization.")
                 sampler = sampler.get_sampler()
 
             if isinstance(domain, Float):
-                if domain.sampler != None:
+                if domain.sampler is not None:
                     logger.warning(
                         "BayesOpt does not support specific sampling methods. "
-                        "The {} sampler will be dropped.".format(
-                            sampler))
+                        "The {} sampler will be dropped.".format(sampler))
                     return (domain.min, domain.max)
 
             raise ValueError("BayesOpt does not support parameters of type "
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 60b2dc4e5cef..692dc61d5f6a 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -164,14 +164,8 @@ def testConvertBayesOpt(self):
         with self.assertRaises(ValueError):
             converted_config = BayesOptSearch.convert_search_space(config)
 
-        config = {
-            "b": {
-                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
-            }
-        }
-        bayesopt_config = {
-            "b/z": (1e-4, 1e-2)
-        }
+        config = {"b": {"z": tune.sample.Float(1e-4, 1e-2).loguniform()}}
+        bayesopt_config = {"b/z": (1e-4, 1e-2)}
 
         converted_config = BayesOptSearch.convert_search_space(config)
 

From e1ba45ca69c33e34218967052b50da6290cb6260 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 31 Aug 2020 14:49:32 +0100
Subject: [PATCH 18/48] Re-added base classes

---
 python/ray/tune/sample.py            | 40 +++++++++++++++++++---------
 python/ray/tune/tests/test_sample.py |  6 ++---
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index da44ae46c406..f0e96cab3b58 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -50,6 +50,22 @@ def sample(self,
         raise NotImplementedError
 
 
+class BaseSampler(Sampler):
+    pass
+
+
+class Uniform(Sampler):
+    pass
+
+
+class LogUniform(Sampler):
+    pass
+
+
+class Normal(Sampler):
+    pass
+
+
 class Grid(Sampler):
     """Dummy sampler used for grid search"""
 
@@ -61,7 +77,7 @@ def sample(self,
 
 
 class Float(Domain):
-    class _Uniform(Sampler):
+    class _Uniform(Uniform):
         def sample(self,
                    domain: "Float",
                    spec: Optional[Union[List[Dict], Dict]] = None,
@@ -75,7 +91,7 @@ def sample(self,
                 return items[0]
             return list(items)
 
-    class _LogUniform(Sampler):
+    class _LogUniform(LogUniform):
         def __init__(self, base: int = 10):
             self.base = base
             assert self.base > 0, "Base has to be strictly greater than 0"
@@ -96,7 +112,7 @@ def sample(self,
                 return items[0]
             return list(items)
 
-    class _Normal(Sampler):
+    class _Normal(Normal):
         def __init__(self, mean: float = 0., sd: float = 0.):
             self.mean = mean
             self.sd = sd
@@ -124,11 +140,6 @@ def __init__(self, min=float("-inf"), max=float("inf")):
         self.min = min
         self.max = max
 
-    def normal(self, mean=0., sd=1.):
-        new = copy(self)
-        new.set_sampler(self._Normal(mean, sd))
-        return new
-
     def uniform(self):
         if not self.min > float("-inf"):
             raise ValueError(
@@ -155,6 +166,11 @@ def loguniform(self, base: int = 10):
         new.set_sampler(self._LogUniform(base))
         return new
 
+    def normal(self, mean=0., sd=1.):
+        new = copy(self)
+        new.set_sampler(self._Normal(mean, sd))
+        return new
+
     def quantized(self, q: Number):
         new = copy(self)
         new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
@@ -162,7 +178,7 @@ def quantized(self, q: Number):
 
 
 class Integer(Domain):
-    class _Uniform(Sampler):
+    class _Uniform(Uniform):
         def sample(self,
                    domain: "Integer",
                    spec: Optional[Union[List[Dict], Dict]] = None,
@@ -190,7 +206,7 @@ def uniform(self):
 
 
 class Categorical(Domain):
-    class _Uniform(Sampler):
+    class _Uniform(Uniform):
         def sample(self,
                    domain: "Categorical",
                    spec: Optional[Union[List[Dict], Dict]] = None,
@@ -225,7 +241,7 @@ def __getitem__(self, item):
 
 
 class Iterative(Domain):
-    class _NextSampler(Sampler):
+    class _NextSampler(BaseSampler):
         def sample(self,
                    domain: "Iterative",
                    spec: Optional[Union[List[Dict], Dict]] = None,
@@ -244,7 +260,7 @@ def __init__(self, iterator: Iterator):
 
 
 class Function(Domain):
-    class _CallSampler(Sampler):
+    class _CallSampler(BaseSampler):
         def sample(self,
                    domain: "Function",
                    spec: Optional[Union[List[Dict], Dict]] = None,
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 692dc61d5f6a..4e786a266606 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -66,12 +66,12 @@ def test_iter():
                 yield i
 
         itr = tune.sample.Iterative(test_iter())
-        samples = itr.uniform().sample(size=5)
+        samples = itr.sample(size=5)
         self.assertTrue(any([-2 <= s <= 2 for s in samples]))
         self.assertTrue(all([c in samples for c in categories]))
 
         itr = tune.sample.Iterative(iter(categories))
-        samples = itr.uniform().sample(size=5)
+        samples = itr.sample(size=5)
         self.assertTrue(any([-2 <= s <= 2 for s in samples]))
         self.assertTrue(all([c in samples for c in categories]))
 
@@ -81,7 +81,7 @@ def sample(spec):
 
         fnc = tune.sample.Function(sample)
 
-        samples = fnc.uniform().sample(size=1000)
+        samples = fnc.sample(size=1000)
         self.assertTrue(any([-4 < s < 4 for s in samples]))
         self.assertTrue(-2 < np.mean(samples) < 2)
 

From 9d32499b5341e1886c218506b87a90838399983b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 31 Aug 2020 14:57:21 +0100
Subject: [PATCH 19/48] Re-factored into list comprehensions

---
 python/ray/tune/suggest/ax.py       | 14 ++++++--------
 python/ray/tune/suggest/bayesopt.py | 10 ++++++----
 python/ray/tune/suggest/optuna.py   | 10 ++++++----
 3 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 74ed7e659b58..cdc86790a730 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -136,14 +136,11 @@ def convert_search_space(spec: Dict):
                 "Grid search parameters cannot be automatically converted "
                 "to an Ax search space.")
 
-        values = []
-
-        for path, val in resolved_vars:
-            values.append({
-                "name": "/".join(path),
-                "type": "fixed",
-                "value": val
-            })
+        values = [{
+            "name": "/".join(path),
+            "type": "fixed",
+            "value": val
+        } for path, val in resolved_vars]
 
         def resolve_value(par, domain):
             sampler = domain.get_sampler()
@@ -199,6 +196,7 @@ def resolve_value(par, domain):
                                  type(domain).__name__,
                                  type(domain.sampler).__name__))
 
+        # Parameter name is e.g. "a/b/c" for nested dicts
         for path, domain in domain_vars:
             par = "/".join(path)
             values.append(resolve_value(par, domain))
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index d731b84af135..99cd0e1ce293 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -322,8 +322,10 @@ def resolve_value(domain):
             raise ValueError("BayesOpt does not support parameters of type "
                              "`{}`".format(type(domain).__name__))
 
-        bounds = {}
-        for path, domain in domain_vars:
-            par = "/".join(path)
-            bounds[par] = resolve_value(domain)
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        bounds = {
+            "/".join(path): resolve_value(domain)
+            for path, domain in domain_vars
+        }
+
         return bounds
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 34d19292dec8..5c22c3cf21fe 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -217,8 +217,10 @@ def resolve_value(par, domain):
                     type(domain).__name__,
                     type(domain.sampler).__name__))
 
-        values = []
-        for path, domain in domain_vars:
-            par = "/".join(path)
-            values.append(resolve_value(par, domain))
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        values = [
+            resolve_value("/".join(path), domain)
+            for path, domain in domain_vars
+        ]
+
         return values

From e46b632200ff1f1645a3085fa24e6994da8dbd42 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 31 Aug 2020 15:49:24 +0100
Subject: [PATCH 20/48] Added `from_config` classmethod for config conversion

---
 python/ray/tune/suggest/ax.py        | 65 +++++++++++++++++++++++++---
 python/ray/tune/suggest/bayesopt.py  | 19 ++++++++
 python/ray/tune/suggest/hyperopt.py  | 18 ++++++++
 python/ray/tune/suggest/optuna.py    | 17 ++++++--
 python/ray/tune/tests/test_sample.py | 16 +++++--
 5 files changed, 121 insertions(+), 14 deletions(-)

diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index cdc86790a730..2d7ba3f9ca04 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -1,5 +1,6 @@
 from typing import Dict
 
+from ax.service.ax_client import AxClient
 from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
     Quantized, Uniform
 from ray.tune.suggest.variant_generator import parse_spec_vars
@@ -32,7 +33,7 @@ class AxSearch(Searcher):
         $ pip install ax-platform sqlalchemy
 
     Parameters:
-        parameters (list[dict]): Parameters in the experiment search space.
+        space (list[dict]): Parameters in the experiment search space.
             Required elements in the dictionaries are: "name" (name of
             this parameter, string), "type" (type of the parameter: "range",
             "fixed", or "choice", string), "bounds" for range parameters
@@ -49,8 +50,11 @@ class AxSearch(Searcher):
             "x3 >= x4" or "x3 + x4 >= 2".
         outcome_constraints (list[str]): Outcome constraints of form
             "metric_name >= bound", like "m1 <= 3."
-        max_concurrent (int): Deprecated.
+        ax_client (AxClient): Optional AxClient instance. If this is set, do
+            not pass any values to these parameters: `space`, `objective_name`,
+            `parameter_constraints`, `outcome_constraints`.
         use_early_stopped_trials: Deprecated.
+        max_concurrent (int): Deprecated.
 
     .. code-block:: python
 
@@ -68,20 +72,58 @@ def easy_objective(config):
                 intermediate_result = config["x1"] + config["x2"] * i
                 tune.report(score=intermediate_result)
 
-        client = AxClient(enforce_sequential_optimization=False)
-        client.create_experiment(parameters=parameters, objective_name="score")
-        algo = AxSearch(client)
+        client = AxClient()
+        algo = AxSearch(space=parameters, objective_name="score")
         tune.run(easy_objective, search_alg=algo)
 
     """
 
     def __init__(self,
-                 ax_client,
+                 space=None,
+                 objective_name=None,
                  mode="max",
+                 parameter_constraints=None,
+                 outcome_constraints=None,
+                 ax_client=None,
                  use_early_stopped_trials=None,
                  max_concurrent=None):
         assert ax is not None, "Ax must be installed!"
+        assert mode in ["min", "max"], "`mode` must be one of ['min', 'max']"
+
+        if not ax_client:
+            ax_client = AxClient()
         self._ax = ax_client
+
+        try:
+            exp = self._ax.experiment
+            has_experiment = True
+        except ValueError:
+            has_experiment = False
+
+        if not has_experiment:
+            if not space:
+                raise ValueError(
+                    "You either have to create an Ax experiment by calling "
+                    "`AxClient.create_experiment()` or you should pass an "
+                    "Ax search space as the `space` parameter to `AxSearch`.")
+            self._ax.create_experiment(
+                parameters=space,
+                objective_name=objective_name,
+                parameter_constraints=parameter_constraints,
+                outcome_constraints=outcome_constraints,
+                minimize=mode != "max")
+        else:
+            if any([
+                    space, objective_name, parameter_constraints,
+                    outcome_constraints
+            ]):
+                raise ValueError(
+                    "If you create the Ax experiment yourself, do not pass "
+                    "values for these parameters to `AxSearch`: {}.".format([
+                        "space", "objective_name", "parameter_constraints",
+                        "outcome_constraints"
+                    ]))
+
         exp = self._ax.experiment
         self._objective_name = exp.optimization_config.objective.metric.name
         self.max_concurrent = max_concurrent
@@ -126,6 +168,17 @@ def _process_result(self, trial_id, result):
         self._ax.complete_trial(
             trial_index=ax_trial_index, raw_data=metric_dict)
 
+    @classmethod
+    def from_config(cls,
+                    config,
+                    objective_name=None,
+                    mode="max",
+                    use_early_stopped_trials=None,
+                    max_concurrent=None):
+        space = cls.convert_search_space(config)
+        return cls(space, objective_name, mode, use_early_stopped_trials,
+                   max_concurrent)
+
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index 99cd0e1ce293..a26bfaf62964 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -289,6 +289,25 @@ def restore(self, checkpoint_path):
              self._total_random_search_trials,
              self._config_counter) = pickle.load(f)
 
+    @classmethod
+    def from_config(cls,
+                    config,
+                    metric,
+                    mode="max",
+                    utility_kwargs=None,
+                    random_state=42,
+                    random_search_steps=10,
+                    verbose=0,
+                    patience=5,
+                    skip_duplicate=True,
+                    analysis=None,
+                    max_concurrent=None,
+                    use_early_stopped_trials=None):
+        space = cls.convert_search_space(config)
+        return cls(space, metric, mode, utility_kwargs, random_state,
+                   random_search_steps, verbose, patience, skip_duplicate,
+                   analysis, max_concurrent, use_early_stopped_trials)
+
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 83e76762f71e..1761a7182ad9 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -244,6 +244,24 @@ def restore(self, checkpoint_path):
         else:
             self.set_state(trials_object)
 
+    @classmethod
+    def from_config(
+            cls,
+            config,
+            metric="episode_reward_mean",
+            mode="max",
+            points_to_evaluate=None,
+            n_initial_points=20,
+            random_state_seed=None,
+            gamma=0.25,
+            max_concurrent=None,
+            use_early_stopped_trials=None,
+    ):
+        space = cls.convert_search_space(config)
+        return cls(space, metric, mode, points_to_evaluate, n_initial_points,
+                   random_state_seed, gamma, max_concurrent,
+                   use_early_stopped_trials)
+
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = copy.deepcopy(spec)
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 5c22c3cf21fe..c9e7aadfa9ea 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -59,8 +59,8 @@ class OptunaSearch(Searcher):
             minimizing or maximizing the metric attribute.
         sampler (optuna.samplers.BaseSampler): Optuna sampler used to
             draw hyperparameter configurations. Defaults to ``TPESampler``.
-        config (dict): Base config dict that gets overwritten by the Optuna
-            sampling and is returned to each Tune trial. This could e.g.
+        base_config (dict): Base config dict that gets overwritten by the
+            Optuna sampling and is returned to each Tune trial. This could e.g.
             contain static variables or configurations that should be passed
             to each trial.
 
@@ -90,7 +90,7 @@ def __init__(
             metric="episode_reward_mean",
             mode="max",
             sampler=None,
-            config=None,
+            base_config=None,
     ):
         assert ot is not None, (
             "Optuna must be installed! Run `pip install optuna`.")
@@ -102,7 +102,7 @@ def __init__(
 
         self._space = space
 
-        self._config = config or {}
+        self._config = base_config or {}
 
         self._study_name = "optuna"  # Fixed study name for in-memory storage
         self._sampler = sampler or ot.samplers.TPESampler()
@@ -162,6 +162,15 @@ def restore(self, checkpoint_path):
         self._storage, self._pruner, self._sampler, \
             self._ot_trials, self._ot_study = save_object
 
+    @classmethod
+    def from_config(cls,
+                    config,
+                    metric="episode_reward_mean",
+                    mode="max",
+                    sampler=None):
+        space = cls.convert_search_space(config)
+        return cls(space, metric, mode, sampler, config)
+
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 4e786a266606..f9aeba99353d 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -134,11 +134,11 @@ def testConvertAx(self):
 
         client1 = AxClient(random_seed=1234)
         client1.create_experiment(parameters=converted_config)
-        searcher1 = AxSearch(client1)
+        searcher1 = AxSearch(ax_client=client1)
 
         client2 = AxClient(random_seed=1234)
         client2.create_experiment(parameters=ax_config)
-        searcher2 = AxSearch(client2)
+        searcher2 = AxSearch(ax_client=client2)
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")
@@ -150,6 +150,8 @@ def testConvertAx(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
+        AxSearch.from_config(config)
+
     def testConvertBayesOpt(self):
         from ray.tune.suggest.bayesopt import BayesOptSearch
 
@@ -179,6 +181,8 @@ def testConvertBayesOpt(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
+        BayesOptSearch.from_config(config, metric="None")
+
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
         from hyperopt import hp
@@ -216,6 +220,8 @@ def testConvertHyperOpt(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
+        HyperOptSearch.from_config(config)
+
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
         from optuna.samplers import RandomSampler
@@ -237,11 +243,11 @@ def testConvertOptuna(self):
 
         sampler1 = RandomSampler(seed=1234)
         searcher1 = OptunaSearch(
-            space=converted_config, sampler=sampler1, config=config)
+            space=converted_config, sampler=sampler1, base_config=config)
 
         sampler2 = RandomSampler(seed=1234)
         searcher2 = OptunaSearch(
-            space=optuna_config, sampler=sampler2, config=config)
+            space=optuna_config, sampler=sampler2, base_config=config)
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")
@@ -253,6 +259,8 @@ def testConvertOptuna(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
+        OptunaSearch.from_config(config)
+
 
 if __name__ == "__main__":
     import pytest

From 0089a633cd9f30aaa22874cb836478dee49df811 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Mon, 31 Aug 2020 19:46:10 +0100
Subject: [PATCH 21/48] Applied suggestions from code review

---
 python/ray/tune/sample.py                    | 40 +++++++-------------
 python/ray/tune/schedulers/pbt.py            |  2 +-
 python/ray/tune/suggest/ax.py                | 23 ++++++-----
 python/ray/tune/suggest/variant_generator.py |  4 +-
 4 files changed, 28 insertions(+), 41 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index f0e96cab3b58..a2508a234035 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -87,9 +87,7 @@ def sample(self,
             assert 0 < domain.max < float("inf"), \
                 "Uniform needs a maximum bound"
             items = np.random.uniform(domain.min, domain.max, size=size)
-            if len(items) == 1:
-                return items[0]
-            return list(items)
+            return items if len(items) > 1 else items[0]
 
     class _LogUniform(LogUniform):
         def __init__(self, base: int = 10):
@@ -108,9 +106,7 @@ def sample(self,
             logmax = np.log(domain.max) / np.log(self.base)
 
             items = self.base**(np.random.uniform(logmin, logmax, size=size))
-            if len(items) == 1:
-                return items[0]
-            return list(items)
+            return items if len(items) > 1 else items[0]
 
     class _Normal(Normal):
         def __init__(self, mean: float = 0., sd: float = 0.):
@@ -130,15 +126,14 @@ def sample(self,
                 loc=self.mean,
                 scale=self.sd)
             items = dist.rvs(size)
-            if len(items) == 1:
-                return items[0]
-            return list(items)
+            return items if len(items) > 1 else items[0]
 
     default_sampler_cls = _Uniform
 
-    def __init__(self, min=float("-inf"), max=float("inf")):
-        self.min = min
-        self.max = max
+    def __init__(self, min: float, max: float):
+        # Need to explicitly check for None
+        self.min = min if min is not None else float("-inf")
+        self.max = max if max is not None else float("inf")
 
     def uniform(self):
         if not self.min > float("-inf"):
@@ -184,9 +179,7 @@ def sample(self,
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
             items = np.random.randint(domain.min, domain.max, size=size)
-            if len(items) == 1:
-                return items[0]
-            return list(items)
+            return items if len(items) > 1 else items[0]
 
     default_sampler_cls = _Uniform
 
@@ -211,12 +204,9 @@ def sample(self,
                    domain: "Categorical",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            choices = []
-            for i in range(size):
-                choices.append(random.choice(domain.categories))
-            if len(choices) == 1:
-                return choices[0]
-            return choices
+
+            items = random.choices(domain.categories, k=size)
+            return items if len(items) > 1 else items[0]
 
     default_sampler_cls = _Uniform
 
@@ -246,12 +236,8 @@ def sample(self,
                    domain: "Iterative",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            items = []
-            for i in range(size):
-                items.append(next(domain.iterator))
-            if len(items) == 1:
-                return items[0]
-            return items
+            items = [next(domain.iterator) for _ in range(size)]
+            return items if len(items) > 1 else items[0]
 
     default_sampler_cls = _NextSampler
 
diff --git a/python/ray/tune/schedulers/pbt.py b/python/ray/tune/schedulers/pbt.py
index 4b84b5d63b4c..40e32e22e387 100644
--- a/python/ray/tune/schedulers/pbt.py
+++ b/python/ray/tune/schedulers/pbt.py
@@ -231,7 +231,7 @@ def __init__(self,
                                (list, dict, Domain)) or callable(value)):
                 raise TypeError("`hyperparam_mutation` values must be either "
                                 "a List, Dict, a tune search space object, or "
-                                "callable.")
+                                "a callable.")
             if type(value) is sample_from:
                 raise ValueError("arbitrary tune.sample_from objects are not "
                                  "supported for `hyperparam_mutation` values."
diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 2d7ba3f9ca04..1f5a61225abe 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -189,12 +189,6 @@ def convert_search_space(spec: Dict):
                 "Grid search parameters cannot be automatically converted "
                 "to an Ax search space.")
 
-        values = [{
-            "name": "/".join(path),
-            "type": "fixed",
-            "value": val
-        } for path, val in resolved_vars]
-
         def resolve_value(par, domain):
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):
@@ -249,8 +243,17 @@ def resolve_value(par, domain):
                                  type(domain).__name__,
                                  type(domain.sampler).__name__))
 
+        # Fixed vars
+        fixed_values = [{
+            "name": "/".join(path),
+            "type": "fixed",
+            "value": val
+        } for path, val in resolved_vars]
+
         # Parameter name is e.g. "a/b/c" for nested dicts
-        for path, domain in domain_vars:
-            par = "/".join(path)
-            values.append(resolve_value(par, domain))
-        return values
+        resolved_values = [
+            resolve_value("/".join(path), domain)
+            for path, domain in domain_vars
+        ]
+
+        return fixed_values + resolved_values
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index 181296faec03..4da49c1fa88c 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -117,9 +117,7 @@ def _clean_value(value):
 
 def parse_spec_vars(spec):
     resolved, unresolved = _split_resolved_unresolved_values(spec)
-    resolved_vars = []
-    for path, value in resolved.items():
-        resolved_vars.append((path, value))
+    resolved_vars = list(resolved.items())
 
     if not unresolved:
         return resolved_vars, [], []

From e8f31c0f9a52facdb129d2e02380360863a6a10f Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 09:43:38 +0100
Subject: [PATCH 22/48] Removed truncated normal distribution

---
 python/ray/tune/sample.py    | 38 ++++++++++++------------------------
 python/requirements_tune.txt |  1 -
 python/setup.py              |  2 +-
 3 files changed, 14 insertions(+), 27 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index a2508a234035..ad14303f26e9 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -90,7 +90,7 @@ def sample(self,
             return items if len(items) > 1 else items[0]
 
     class _LogUniform(LogUniform):
-        def __init__(self, base: int = 10):
+        def __init__(self, base: float = 10):
             self.base = base
             assert self.base > 0, "Base has to be strictly greater than 0"
 
@@ -119,18 +119,16 @@ def sample(self,
                    domain: "Float",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            # Use a truncated normal to avoid oversampling border values
-            dist = stats.truncnorm(
-                (domain.min - self.mean) / self.sd,
-                (domain.max - self.mean) / self.sd,
-                loc=self.mean,
-                scale=self.sd)
-            items = dist.rvs(size)
+            assert not domain.min or domain.min == float("-inf"), \
+                "Normal sampling does not allow a lower value bound."
+            assert not domain.max or domain.max == float("inf"), \
+                "Normal sampling does not allow a upper value bound."
+            items = np.random.normal(self.mean, self.sd, size=size)
             return items if len(items) > 1 else items[0]
 
     default_sampler_cls = _Uniform
 
-    def __init__(self, min: float, max: float):
+    def __init__(self, min: Optional[float], max: Optional[float]):
         # Need to explicitly check for None
         self.min = min if min is not None else float("-inf")
         self.max = max if max is not None else float("inf")
@@ -148,7 +146,7 @@ def uniform(self):
         new.set_sampler(self._Uniform())
         return new
 
-    def loguniform(self, base: int = 10):
+    def loguniform(self, base: float = 10):
         if not self.min > 0:
             raise ValueError(
                 "LogUniform requires a minimum bound greater than 0. "
@@ -332,7 +330,7 @@ def loguniform(min: float, max: float, base: float = 10):
     return Float(min, max).loguniform(base)
 
 
-def qloguniform(min, max, q, base=10):
+def qloguniform(min: float, max: float, q, base=10):
     """Sugar for sampling in different orders of magnitude.
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
@@ -385,14 +383,9 @@ def qrandint(min: int, max: int, q: int = 1):
 
 
 def randn(mean: float = 0.,
-          sd: float = 1.,
-          min: float = float("-inf"),
-          max: float = float("inf")):
+          sd: float = 1.):
     """Sample a float value normally with ``mean`` and ``sd``.
 
-    Will truncate the normal distribution at ``min`` and ``max`` to avoid
-    oversampling the border regions.
-
     Args:
         mean (float): Mean of the normal distribution. Defaults to 0.
         sd (float): SD of the normal distribution. Defaults to 1.
@@ -400,21 +393,16 @@ def randn(mean: float = 0.,
         max (float): Maximum bound. Defaults to inf.
 
     """
-    return Float(min, max).normal(mean, sd)
+    return Float(None, None).normal(mean, sd)
 
 
 def qrandn(q: float,
            mean: float = 0.,
-           sd: float = 1.,
-           min: float = float("-inf"),
-           max: float = float("inf")):
+           sd: float = 1.):
     """Sample a float value normally with ``mean`` and ``sd``.
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
 
-    Will truncate the normal distribution at ``min`` and ``max`` to avoid
-    oversampling the border regions.
-
     Args:
         q (float): Quantization number. The result will be rounded to an
             integer increment of this value.
@@ -424,4 +412,4 @@ def qrandn(q: float,
         max (float): Maximum bound. Defaults to inf.
 
     """
-    return Float(min, max).normal(mean, sd).quantized(q)
+    return Float(None, None).normal(mean, sd).quantized(q)
diff --git a/python/requirements_tune.txt b/python/requirements_tune.txt
index b61dc62393a4..1123fc7ed879 100644
--- a/python/requirements_tune.txt
+++ b/python/requirements_tune.txt
@@ -19,7 +19,6 @@ optuna
 pytest-remotedata>=0.3.1
 pytorch-lightning
 scikit-optimize
-scipy
 sigopt
 smart_open
 tensorflow_probability
diff --git a/python/setup.py b/python/setup.py
index 0d804209af06..7aeeaab40b7b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -111,7 +111,7 @@
 extras = {
     "debug": [],
     "serve": ["uvicorn", "flask", "requests"],
-    "tune": ["tabulate", "tensorboardX", "pandas", "scipy"]
+    "tune": ["tabulate", "tensorboardX", "pandas"]
 }
 
 extras["rllib"] = extras["tune"] + [

From e4b404fb4213d73bbd2bf32cf91c34fff5c28e86 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 10:55:01 +0100
Subject: [PATCH 23/48] Set search properties in tune.run

---
 python/ray/tune/sample.py                   |  8 +-
 python/ray/tune/suggest/ax.py               | 86 +++++++++++++--------
 python/ray/tune/suggest/bayesopt.py         | 64 +++++++++------
 python/ray/tune/suggest/hyperopt.py         | 47 ++++++-----
 python/ray/tune/suggest/optuna.py           | 33 +++++---
 python/ray/tune/suggest/search.py           | 17 ++++
 python/ray/tune/suggest/search_generator.py |  3 +
 python/ray/tune/suggest/suggestion.py       | 16 ++++
 python/ray/tune/tests/test_sample.py        | 24 +++---
 python/ray/tune/tune.py                     | 11 +++
 10 files changed, 206 insertions(+), 103 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index ad14303f26e9..c2e917e3f381 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -7,7 +7,6 @@
     Union
 
 import numpy as np
-from scipy import stats
 
 logger = logging.getLogger(__name__)
 
@@ -382,8 +381,7 @@ def qrandint(min: int, max: int, q: int = 1):
     return Integer(min, max).uniform().quantized(q)
 
 
-def randn(mean: float = 0.,
-          sd: float = 1.):
+def randn(mean: float = 0., sd: float = 1.):
     """Sample a float value normally with ``mean`` and ``sd``.
 
     Args:
@@ -396,9 +394,7 @@ def randn(mean: float = 0.,
     return Float(None, None).normal(mean, sd)
 
 
-def qrandn(q: float,
-           mean: float = 0.,
-           sd: float = 1.):
+def qrandn(q: float, mean: float = 0., sd: float = 1.):
     """Sample a float value normally with ``mean`` and ``sd``.
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 1f5a61225abe..6fa466ec62f9 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -80,7 +80,7 @@ def easy_objective(config):
 
     def __init__(self,
                  space=None,
-                 objective_name=None,
+                 metric="episode_reward_mean",
                  mode="max",
                  parameter_constraints=None,
                  outcome_constraints=None,
@@ -90,9 +90,29 @@ def __init__(self,
         assert ax is not None, "Ax must be installed!"
         assert mode in ["min", "max"], "`mode` must be one of ['min', 'max']"
 
-        if not ax_client:
-            ax_client = AxClient()
+        super(AxSearch, self).__init__(
+            metric=metric,
+            mode=mode,
+            max_concurrent=max_concurrent,
+            use_early_stopped_trials=use_early_stopped_trials)
+
         self._ax = ax_client
+        self._space = space
+        self._parameter_constraints = parameter_constraints
+        self._outcome_constraints = outcome_constraints
+
+        self.max_concurrent = max_concurrent
+
+        self._objective_name = metric
+        self._parameters = []
+        self._live_trial_mapping = {}
+
+        if self._space:
+            self.setup_experiment()
+
+    def setup_experiment(self):
+        if not self._ax:
+            self._ax = AxClient()
 
         try:
             exp = self._ax.experiment
@@ -101,44 +121,55 @@ def __init__(self,
             has_experiment = False
 
         if not has_experiment:
-            if not space:
+            if not self._space:
                 raise ValueError(
-                    "You either have to create an Ax experiment by calling "
-                    "`AxClient.create_experiment()` or you should pass an "
-                    "Ax search space as the `space` parameter to `AxSearch`.")
+                    "You have to create an Ax experiment by calling "
+                    "`AxClient.create_experiment()`, or you should pass an "
+                    "Ax search space as the `space` parameter to `AxSearch`, "
+                    "or pass a `config` dict to `tune.run()`.")
             self._ax.create_experiment(
-                parameters=space,
-                objective_name=objective_name,
-                parameter_constraints=parameter_constraints,
-                outcome_constraints=outcome_constraints,
-                minimize=mode != "max")
+                parameters=self._space,
+                objective_name=self._metric,
+                parameter_constraints=self._parameter_constraints,
+                outcome_constraints=self._outcome_constraints,
+                minimize=self._mode != "max")
         else:
             if any([
-                    space, objective_name, parameter_constraints,
-                    outcome_constraints
+                    self._space, self._metric, self._parameter_constraints,
+                    self._outcome_constraints
             ]):
                 raise ValueError(
                     "If you create the Ax experiment yourself, do not pass "
                     "values for these parameters to `AxSearch`: {}.".format([
-                        "space", "objective_name", "parameter_constraints",
+                        "space", "metric", "parameter_constraints",
                         "outcome_constraints"
                     ]))
 
         exp = self._ax.experiment
         self._objective_name = exp.optimization_config.objective.metric.name
-        self.max_concurrent = max_concurrent
         self._parameters = list(exp.parameters)
-        self._live_trial_mapping = {}
-        super(AxSearch, self).__init__(
-            metric=self._objective_name,
-            mode=mode,
-            max_concurrent=max_concurrent,
-            use_early_stopped_trials=use_early_stopped_trials)
+
         if self._ax._enforce_sequential_optimization:
             logger.warning("Detected sequential enforcement. Be sure to use "
                            "a ConcurrencyLimiter.")
 
+    def set_search_properties(self, metric, mode, config):
+        if self._ax:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        self._metric = metric
+        self._mode = mode
+        self.setup_experiment()
+
     def suggest(self, trial_id):
+        if not self._ax:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:
                 return None
@@ -168,17 +199,6 @@ def _process_result(self, trial_id, result):
         self._ax.complete_trial(
             trial_index=ax_trial_index, raw_data=metric_dict)
 
-    @classmethod
-    def from_config(cls,
-                    config,
-                    objective_name=None,
-                    mode="max",
-                    use_early_stopped_trials=None,
-                    max_concurrent=None):
-        space = cls.convert_search_space(config)
-        return cls(space, objective_name, mode, use_early_stopped_trials,
-                   max_concurrent)
-
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index a26bfaf62964..9248c87b441c 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -81,8 +81,8 @@ class BayesOptSearch(Searcher):
     optimizer = None
 
     def __init__(self,
-                 space,
-                 metric,
+                 space=None,
+                 metric="episode_reward_mean",
                  mode="max",
                  utility_kwargs=None,
                  random_state=42,
@@ -159,15 +159,43 @@ def __init__(self,
         self.random_search_trials = random_search_steps
         self._total_random_search_trials = 0
 
-        self.optimizer = byo.BayesianOptimization(
-            f=None, pbounds=space, verbose=verbose, random_state=random_state)
-
         self.utility = byo.UtilityFunction(**utility_kwargs)
 
         # Registering the provided analysis, if given
         if analysis is not None:
             self.register_analysis(analysis)
 
+        self._space = space
+        self._verbose = verbose
+        self._random_state = random_state
+
+        self.optimizer = None
+        if space:
+            self.setup_optimizer()
+
+    def setup_optimizer(self):
+        self.optimizer = byo.BayesianOptimization(
+            f=None,
+            pbounds=self._space,
+            verbose=self._verbose,
+            random_state=self._random_state)
+
+    def set_search_properties(self, metric, mode, config):
+        if self.optimizer:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        self._metric = metric
+        self._mode = mode
+
+        if mode == "max":
+            self._metric_op = 1.
+        elif mode == "min":
+            self._metric_op = -1.
+
+        self.setup_optimizer()
+        return True
+
     def suggest(self, trial_id):
         """Return new point to be explored by black box function.
 
@@ -179,6 +207,13 @@ def suggest(self, trial_id):
             Either a dictionary describing the new point to explore or
             None, when no new point is to be explored for the time being.
         """
+        if not self.optimizer:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         # If we have more active trials than the allowed maximum
         total_live_trials = len(self._live_trial_mapping)
         if self.max_concurrent and self.max_concurrent <= total_live_trials:
@@ -289,25 +324,6 @@ def restore(self, checkpoint_path):
              self._total_random_search_trials,
              self._config_counter) = pickle.load(f)
 
-    @classmethod
-    def from_config(cls,
-                    config,
-                    metric,
-                    mode="max",
-                    utility_kwargs=None,
-                    random_state=42,
-                    random_search_steps=10,
-                    verbose=0,
-                    patience=5,
-                    skip_duplicate=True,
-                    analysis=None,
-                    max_concurrent=None,
-                    use_early_stopped_trials=None):
-        space = cls.convert_search_space(config)
-        return cls(space, metric, mode, utility_kwargs, random_state,
-                   random_search_steps, verbose, patience, skip_duplicate,
-                   analysis, max_concurrent, use_early_stopped_trials)
-
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 1761a7182ad9..c81c642d8455 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -92,7 +92,7 @@ class HyperOptSearch(Searcher):
 
     def __init__(
             self,
-            space,
+            space=None,
             metric="episode_reward_mean",
             mode="max",
             points_to_evaluate=None,
@@ -124,7 +124,6 @@ def __init__(
                 hpo.tpe.suggest, n_startup_jobs=n_initial_points)
         if gamma is not None:
             self.algo = partial(self.algo, gamma=gamma)
-        self.domain = hpo.Domain(lambda spc: spc, space)
         if points_to_evaluate is None:
             self._hpopt_trials = hpo.Trials()
             self._points_to_evaluate = 0
@@ -140,7 +139,33 @@ def __init__(
         else:
             self.rstate = np.random.RandomState(random_state_seed)
 
+        self.domain = None
+        if space:
+            self.domain = hpo.Domain(lambda spc: spc, space)
+
+    def set_search_properties(self, metric, mode, config):
+        if self.domain:
+            return False
+        space = self.convert_search_space(config)
+        self.domain = hpo.Domain(lambda spc: spc, space)
+
+        self._metric = metric
+        self._mode = mode
+
+        if mode == "max":
+            self.metric_op = -1.
+        elif mode == "min":
+            self.metric_op = 1.
+
+        return True
+
     def suggest(self, trial_id):
+        if not self.domain:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:
                 return None
@@ -244,24 +269,6 @@ def restore(self, checkpoint_path):
         else:
             self.set_state(trials_object)
 
-    @classmethod
-    def from_config(
-            cls,
-            config,
-            metric="episode_reward_mean",
-            mode="max",
-            points_to_evaluate=None,
-            n_initial_points=20,
-            random_state_seed=None,
-            gamma=0.25,
-            max_concurrent=None,
-            use_early_stopped_trials=None,
-    ):
-        space = cls.convert_search_space(config)
-        return cls(space, metric, mode, points_to_evaluate, n_initial_points,
-                   random_state_seed, gamma, max_concurrent,
-                   use_early_stopped_trials)
-
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = copy.deepcopy(spec)
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index c9e7aadfa9ea..ebec4dbde6fe 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -86,7 +86,7 @@ class OptunaSearch(Searcher):
 
     def __init__(
             self,
-            space,
+            space=None,
             metric="episode_reward_mean",
             mode="max",
             sampler=None,
@@ -114,6 +114,11 @@ def __init__(
         self._storage = ot.storages.InMemoryStorage()
 
         self._ot_trials = {}
+        self._ot_study = None
+        if self._space:
+            self.setup_study(mode)
+
+    def setup_study(self, mode):
         self._ot_study = ot.study.create_study(
             storage=self._storage,
             sampler=self._sampler,
@@ -122,7 +127,24 @@ def __init__(
             direction="minimize" if mode == "min" else "maximize",
             load_if_exists=True)
 
+    def set_search_properties(self, metric, mode, config):
+        if self._space:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        self._metric = metric
+        self._mode = mode
+        self.setup_study(mode)
+        self._config = config
+
     def suggest(self, trial_id):
+        if not self._space:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         if trial_id not in self._ot_trials:
             ot_trial_id = self._storage.create_new_trial(
                 self._ot_study._study_id)
@@ -162,15 +184,6 @@ def restore(self, checkpoint_path):
         self._storage, self._pruner, self._sampler, \
             self._ot_trials, self._ot_study = save_object
 
-    @classmethod
-    def from_config(cls,
-                    config,
-                    metric="episode_reward_mean",
-                    mode="max",
-                    sampler=None):
-        space = cls.convert_search_space(config)
-        return cls(space, metric, mode, sampler, config)
-
     @staticmethod
     def convert_search_space(spec: Dict):
         spec = flatten_dict(spec)
diff --git a/python/ray/tune/suggest/search.py b/python/ray/tune/suggest/search.py
index 64562753384e..a878410a95f9 100644
--- a/python/ray/tune/suggest/search.py
+++ b/python/ray/tune/suggest/search.py
@@ -12,6 +12,23 @@ class SearchAlgorithm:
     """
     _finished = False
 
+    def set_search_properties(self, metric, mode, config):
+        """Pass search properties to search algorithm.
+
+        This method acts as an alternative to instantiating search algorithms
+        with their own specific search spaces. Instead they can accept a
+        Tune config through this method.
+
+        The search algorithm will usually pass this method to their
+        ``Searcher`` instance.
+
+        Args:
+            metric (str): Metric to optimize
+            mode (str): One of ["min", "max"]. Direction to optimize.
+            config (dict): Tune config dict.
+        """
+        return True
+
     def add_configurations(self, experiments):
         """Tracks given experiment specifications.
 
diff --git a/python/ray/tune/suggest/search_generator.py b/python/ray/tune/suggest/search_generator.py
index bc23247d904a..c5990c63c5c1 100644
--- a/python/ray/tune/suggest/search_generator.py
+++ b/python/ray/tune/suggest/search_generator.py
@@ -69,6 +69,9 @@ def __init__(self, searcher):
         self._total_samples = None  # int: total samples to evaluate.
         self._finished = False
 
+    def set_search_properties(self, metric, mode, config):
+        return self.searcher.set_search_properties(metric, mode, config)
+
     def add_configurations(self, experiments):
         """Registers experiment specifications.
 
diff --git a/python/ray/tune/suggest/suggestion.py b/python/ray/tune/suggest/suggestion.py
index e3cee6dfb7e6..95e5bea8efff 100644
--- a/python/ray/tune/suggest/suggestion.py
+++ b/python/ray/tune/suggest/suggestion.py
@@ -69,6 +69,22 @@ def __init__(self,
         self._metric = metric
         self._mode = mode
 
+    def set_search_properties(self, metric, mode, config):
+        """Pass search properties to searcher.
+
+        This method acts as an alternative to instantiating search algorithms
+        with their own specific search spaces. Instead they can accept a
+        Tune config through this method. A searcher should return ``True``
+        if setting the config was successful, or ``False`` if it was
+        unsuccessful, e.g. when the search space has already been set.
+
+        Args:
+            metric (str): Metric to optimize
+            mode (str): One of ["min", "max"]. Direction to optimize.
+            config (dict): Tune config dict.
+        """
+        return True
+
     def on_trial_result(self, trial_id, result):
         """Optional notification for result during training.
 
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index f9aeba99353d..42d1187faf6a 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -18,11 +18,6 @@ def testBoundedFloat(self):
         with self.assertRaises(ValueError):
             bounded.normal().uniform()
 
-        # Normal
-        samples = bounded.normal(-4, 2).sample(size=1000)
-        self.assertTrue(any(-4.2 < s < 8.3 for s in samples))
-        self.assertTrue(np.mean(samples) < -2)
-
         # Uniform
         samples = bounded.uniform().sample(size=1000)
         self.assertTrue(any(-4.2 < s < 8.3 for s in samples))
@@ -37,12 +32,17 @@ def testBoundedFloat(self):
         self.assertTrue(any(1e-4 < s < 1e-1 for s in samples))
 
     def testUnboundedFloat(self):
-        unbounded = tune.sample.Float()
+        unbounded = tune.sample.Float(None, None)
 
         # Require min and max bounds for loguniform
         with self.assertRaises(ValueError):
             unbounded.loguniform()
 
+        # Normal
+        samples = tune.sample.Float(None, None).normal().sample(size=1000)
+        self.assertTrue(any(-5 < s < 5 for s in samples))
+        self.assertTrue(-1 < np.mean(samples) < 1)
+
     def testBoundedInt(self):
         bounded = tune.sample.Integer(-3, 12)
 
@@ -150,7 +150,8 @@ def testConvertAx(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
-        AxSearch.from_config(config)
+        searcher = AxSearch()
+        searcher.set_search_properties("none", "max", config)
 
     def testConvertBayesOpt(self):
         from ray.tune.suggest.bayesopt import BayesOptSearch
@@ -181,7 +182,8 @@ def testConvertBayesOpt(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
-        BayesOptSearch.from_config(config, metric="None")
+        searcher = BayesOptSearch()
+        searcher.set_search_properties("none", "max", config)
 
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
@@ -220,7 +222,8 @@ def testConvertHyperOpt(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
-        HyperOptSearch.from_config(config)
+        searcher = HyperOptSearch()
+        searcher.set_search_properties("none", "max", config)
 
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
@@ -259,7 +262,8 @@ def testConvertOptuna(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
-        OptunaSearch.from_config(config)
+        searcher = OptunaSearch()
+        searcher.set_search_properties("none", "max", config)
 
 
 if __name__ == "__main__":
diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py
index 543de99a51c5..b5800d73408d 100644
--- a/python/ray/tune/tune.py
+++ b/python/ray/tune/tune.py
@@ -67,6 +67,8 @@ def _report_progress(runner, reporter, done=False):
 
 def run(run_or_experiment,
         name=None,
+        metric="episode_reward_mean",
+        mode="max",
         stop=None,
         config=None,
         resources_per_trial=None,
@@ -144,6 +146,8 @@ def run(run_or_experiment,
             ``tune.register_trainable("lambda_id", lambda x: ...)``. You can
             then use ``tune.run("lambda_id")``.
         name (str): Name of experiment.
+        metric (str): Metric to optimize.
+        mode (str): One one ["min", "max"]. Direction to optimize.
         stop (dict | callable | :class:`Stopper`): Stopping criteria. If dict,
             the keys may be any field in the return result of 'train()',
             whichever is reached first. If function, it must take (trial_id,
@@ -328,6 +332,13 @@ def run(run_or_experiment,
     if not search_alg:
         search_alg = BasicVariantGenerator()
 
+    if config and not search_alg.set_search_properties(metric, mode, config):
+        logger.warning(
+            "You passed a `config` parameter to `tune.run()`, but the "
+            "search algorithm was already instantiated with a search space. "
+            "Any search definitions in the `config` passed to `tune.run()` "
+            "will be ignored.")
+
     runner = TrialRunner(
         search_alg=search_alg,
         scheduler=scheduler or FIFOScheduler(),

From 8e74a1b035e541b98c32848f2035351cc8a20e1b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 11:05:30 +0100
Subject: [PATCH 24/48] Added test for tune.run search properties

---
 python/ray/tune/sample.py            | 12 +++++---
 python/ray/tune/suggest/ax.py        |  1 +
 python/ray/tune/suggest/optuna.py    |  1 +
 python/ray/tune/tests/test_sample.py | 44 +++++++++++++++++++++++++---
 4 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index c2e917e3f381..531143979bcb 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -50,19 +50,23 @@ def sample(self,
 
 
 class BaseSampler(Sampler):
-    pass
+    def __str__(self):
+        return "Base"
 
 
 class Uniform(Sampler):
-    pass
+    def __str__(self):
+        return "Uniform"
 
 
 class LogUniform(Sampler):
-    pass
+    def __str__(self):
+        return "LogUniform"
 
 
 class Normal(Sampler):
-    pass
+    def __str__(self):
+        return "Normal"
 
 
 class Grid(Sampler):
diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 6fa466ec62f9..cb23a4c581b5 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -161,6 +161,7 @@ def set_search_properties(self, metric, mode, config):
         self._metric = metric
         self._mode = mode
         self.setup_experiment()
+        return True
 
     def suggest(self, trial_id):
         if not self._ax:
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index ebec4dbde6fe..db1d02590367 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -136,6 +136,7 @@ def set_search_properties(self, metric, mode, config):
         self._mode = mode
         self.setup_study(mode)
         self._config = config
+        return True
 
     def suggest(self, trial_id):
         if not self._space:
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 42d1187faf6a..f9938ddf5d31 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -4,6 +4,10 @@
 from ray import tune
 
 
+def _mock_objective(config):
+    tune.report(**config)
+
+
 class SearchSpaceTest(unittest.TestCase):
     def setUp(self):
         pass
@@ -151,7 +155,15 @@ def testConvertAx(self):
         self.assertLess(config1["b"]["z"], 1e-2)
 
         searcher = AxSearch()
-        searcher.set_search_properties("none", "max", config)
+        analysis = tune.run(
+            _mock_objective,
+            metric="a",
+            mode="max",
+            config=config,
+            search_alg=searcher,
+            num_samples=1)
+        trial = analysis.trials[0]
+        assert trial.config["a"] in [2, 3, 4]
 
     def testConvertBayesOpt(self):
         from ray.tune.suggest.bayesopt import BayesOptSearch
@@ -183,7 +195,15 @@ def testConvertBayesOpt(self):
         self.assertLess(config1["b"]["z"], 1e-2)
 
         searcher = BayesOptSearch()
-        searcher.set_search_properties("none", "max", config)
+        analysis = tune.run(
+            _mock_objective,
+            metric="a",
+            mode="max",
+            config=config,
+            search_alg=searcher,
+            num_samples=1)
+        trial = analysis.trials[0]
+        self.assertLess(trial.config["b"]["z"], 1e-2)
 
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
@@ -223,7 +243,15 @@ def testConvertHyperOpt(self):
         self.assertLess(config1["b"]["z"], 1e-2)
 
         searcher = HyperOptSearch()
-        searcher.set_search_properties("none", "max", config)
+        analysis = tune.run(
+            _mock_objective,
+            metric="a",
+            mode="max",
+            config=config,
+            search_alg=searcher,
+            num_samples=1)
+        trial = analysis.trials[0]
+        assert trial.config["a"] in [2, 3, 4]
 
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
@@ -263,7 +291,15 @@ def testConvertOptuna(self):
         self.assertLess(config1["b"]["z"], 1e-2)
 
         searcher = OptunaSearch()
-        searcher.set_search_properties("none", "max", config)
+        analysis = tune.run(
+            _mock_objective,
+            metric="a",
+            mode="max",
+            config=config,
+            search_alg=searcher,
+            num_samples=1)
+        trial = analysis.trials[0]
+        assert trial.config["a"] in [2, 3, 4]
 
 
 if __name__ == "__main__":

From 77c3cd78eb4e79d34942bffe06660cea511ddc29 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 12:18:48 +0100
Subject: [PATCH 25/48] Move sampler initializers to base classes

---
 python/ray/tune/sample.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 531143979bcb..bd68b6955dfb 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -60,11 +60,21 @@ def __str__(self):
 
 
 class LogUniform(Sampler):
+    def __init__(self, base: float = 10):
+        self.base = base
+        assert self.base > 0, "Base has to be strictly greater than 0"
+
     def __str__(self):
         return "LogUniform"
 
 
 class Normal(Sampler):
+    def __init__(self, mean: float = 0., sd: float = 0.):
+        self.mean = mean
+        self.sd = sd
+
+        assert self.sd > 0, "SD has to be strictly greater than 0"
+
     def __str__(self):
         return "Normal"
 
@@ -93,10 +103,6 @@ def sample(self,
             return items if len(items) > 1 else items[0]
 
     class _LogUniform(LogUniform):
-        def __init__(self, base: float = 10):
-            self.base = base
-            assert self.base > 0, "Base has to be strictly greater than 0"
-
         def sample(self,
                    domain: "Float",
                    spec: Optional[Union[List[Dict], Dict]] = None,
@@ -112,12 +118,6 @@ def sample(self,
             return items if len(items) > 1 else items[0]
 
     class _Normal(Normal):
-        def __init__(self, mean: float = 0., sd: float = 0.):
-            self.mean = mean
-            self.sd = sd
-
-            assert self.sd > 0, "SD has to be strictly greater than 0"
-
         def sample(self,
                    domain: "Float",
                    spec: Optional[Union[List[Dict], Dict]] = None,

From 3642e2d3aa0f409285c784ff9ada4c673f425c75 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 12:58:58 +0100
Subject: [PATCH 26/48] Add tune API sampling test, fixed includes, fixed
 resampling bug

---
 python/ray/tune/__init__.py                  |  9 ++--
 python/ray/tune/sample.py                    | 44 +++++++++------
 python/ray/tune/suggest/variant_generator.py |  2 +
 python/ray/tune/tests/test_sample.py         | 56 ++++++++++++++++++++
 4 files changed, 90 insertions(+), 21 deletions(-)

diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py
index 6312fe8986dc..e41544e7063e 100644
--- a/python/ray/tune/__init__.py
+++ b/python/ray/tune/__init__.py
@@ -12,14 +12,15 @@
                               save_checkpoint, checkpoint_dir)
 from ray.tune.progress_reporter import (ProgressReporter, CLIReporter,
                                         JupyterNotebookReporter)
-from ray.tune.sample import (sample_from, uniform, choice, randint, randn,
-                             loguniform)
+from ray.tune.sample import (sample_from, uniform, quniform, choice, randint,
+                             qrandint, randn, qrandn, loguniform, qloguniform)
 
 __all__ = [
     "Trainable", "DurableTrainable", "TuneError", "grid_search",
     "register_env", "register_trainable", "run", "run_experiments", "Stopper",
-    "EarlyStopping", "Experiment", "sample_from", "track", "uniform", "choice",
-    "randint", "randn", "loguniform", "ExperimentAnalysis", "Analysis",
+    "EarlyStopping", "Experiment", "sample_from", "track", "uniform",
+    "quniform", "choice", "randint", "qrandint", "randn", "qrandn",
+    "loguniform", "qloguniform", "ExperimentAnalysis", "Analysis",
     "CLIReporter", "JupyterNotebookReporter", "ProgressReporter", "report",
     "get_trial_dir", "get_trial_name", "get_trial_id", "make_checkpoint_dir",
     "save_checkpoint", "checkpoint_dir"
diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index bd68b6955dfb..9ab7ee7b9850 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -15,6 +15,10 @@ class Domain:
     sampler = None
     default_sampler_cls = None
 
+    def cast(self, value):
+        """Cast value to domain type"""
+        return value
+
     def set_sampler(self, sampler, allow_override=False):
         if self.sampler and not allow_override:
             raise ValueError("You can only choose one sampler for parameter "
@@ -97,10 +101,10 @@ def sample(self,
                    size: int = 1):
             assert domain.min > float("-inf"), \
                 "Uniform needs a minimum bound"
-            assert 0 < domain.max < float("inf"), \
+            assert domain.max < float("inf"), \
                 "Uniform needs a maximum bound"
             items = np.random.uniform(domain.min, domain.max, size=size)
-            return items if len(items) > 1 else items[0]
+            return items if len(items) > 1 else domain.cast(items[0])
 
     class _LogUniform(LogUniform):
         def sample(self,
@@ -115,7 +119,7 @@ def sample(self,
             logmax = np.log(domain.max) / np.log(self.base)
 
             items = self.base**(np.random.uniform(logmin, logmax, size=size))
-            return items if len(items) > 1 else items[0]
+            return items if len(items) > 1 else domain.cast(items[0])
 
     class _Normal(Normal):
         def sample(self,
@@ -127,7 +131,7 @@ def sample(self,
             assert not domain.max or domain.max == float("inf"), \
                 "Normal sampling does not allow a upper value bound."
             items = np.random.normal(self.mean, self.sd, size=size)
-            return items if len(items) > 1 else items[0]
+            return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _Uniform
 
@@ -136,6 +140,9 @@ def __init__(self, min: Optional[float], max: Optional[float]):
         self.min = min if min is not None else float("-inf")
         self.max = max if max is not None else float("inf")
 
+    def cast(self, value):
+        return float(value)
+
     def uniform(self):
         if not self.min > float("-inf"):
             raise ValueError(
@@ -180,7 +187,7 @@ def sample(self,
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
             items = np.random.randint(domain.min, domain.max, size=size)
-            return items if len(items) > 1 else items[0]
+            return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _Uniform
 
@@ -188,6 +195,9 @@ def __init__(self, min, max):
         self.min = min
         self.max = max
 
+    def cast(self, value):
+        return int(value)
+
     def quantized(self, q: Number):
         new = copy(self)
         new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
@@ -207,7 +217,7 @@ def sample(self,
                    size: int = 1):
 
             items = random.choices(domain.categories, k=size)
-            return items if len(items) > 1 else items[0]
+            return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _Uniform
 
@@ -238,7 +248,7 @@ def sample(self,
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
             items = [next(domain.iterator) for _ in range(size)]
-            return items if len(items) > 1 else items[0]
+            return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _NextSampler
 
@@ -285,8 +295,8 @@ def sample(self,
                size: int = 1):
         values = self.sampler.sample(domain, spec, size)
         quantized = np.round(np.divide(values, self.q)) * self.q
-        if len(quantized) == 1:
-            return quantized[0]
+        if not isinstance(quantized, np.ndarray):
+            return domain.cast(quantized)
         return list(quantized)
 
 
@@ -316,6 +326,7 @@ def quniform(min: float, max: float, q: float):
     ``np.random.uniform(1, 10))``
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
+        Quantization makes the upper bound inclusive.
 
     """
     return Float(min, max).uniform().quantized(q)
@@ -338,6 +349,8 @@ def qloguniform(min: float, max: float, q, base=10):
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
 
+    Quantization makes the upper bound inclusive.
+
     Args:
         min (float): Lower boundary of the output interval (e.g. 1e-4)
         max (float): Upper boundary of the output interval (e.g. 1e-2)
@@ -374,9 +387,10 @@ def randint(min: int, max: int):
 def qrandint(min: int, max: int, q: int = 1):
     """Sample an integer value uniformly between ``min`` and ``max``.
 
-    ``min`` is inclusive, ``max`` is exclusive.
+    ``min`` is inclusive, ``max`` is also inclusive (!).
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
+    Quantization makes the upper bound inclusive.
 
     Sampling from ``tune.randint(10)`` is equivalent to sampling from
     ``np.random.randint(10)``
@@ -391,25 +405,21 @@ def randn(mean: float = 0., sd: float = 1.):
     Args:
         mean (float): Mean of the normal distribution. Defaults to 0.
         sd (float): SD of the normal distribution. Defaults to 1.
-        min (float): Minimum bound. Defaults to -inf.
-        max (float): Maximum bound. Defaults to inf.
 
     """
     return Float(None, None).normal(mean, sd)
 
 
-def qrandn(q: float, mean: float = 0., sd: float = 1.):
+def qrandn(mean: float, sd: float, q: float):
     """Sample a float value normally with ``mean`` and ``sd``.
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
 
     Args:
+        mean (float): Mean of the normal distribution.
+        sd (float): SD of the normal distribution.
         q (float): Quantization number. The result will be rounded to an
             integer increment of this value.
-        mean (float): Mean of the normal distribution. Defaults to 0.
-        sd (float): SD of the normal distribution. Defaults to 1.
-        min (float): Minimum bound. Defaults to -inf.
-        max (float): Maximum bound. Defaults to inf.
 
     """
     return Float(None, None).normal(mean, sd).quantized(q)
diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index 4da49c1fa88c..5437a944db8a 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -179,6 +179,8 @@ def _resolve_domain_vars(spec, domain_vars):
         num_passes += 1
         error = False
         for path, domain in domain_vars:
+            if path in resolved:
+                continue
             try:
                 value = domain.sample(_UnresolvedAccessGuard(spec))
             except RecursiveDependencyError as e:
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index f9938ddf5d31..c55953d38ce5 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -2,6 +2,7 @@
 import unittest
 
 from ray import tune
+from ray.tune.suggest.variant_generator import generate_variants
 
 
 def _mock_objective(config):
@@ -15,6 +16,61 @@ def setUp(self):
     def tearDown(self):
         pass
 
+    def testTuneSampleAPI(self):
+        config = {
+            "func": tune.sample_from(lambda spec: spec.config.uniform * 0.01),
+            "uniform": tune.uniform(-5, -1),
+            "quniform": tune.quniform(3.2, 5.4, 0.2),
+            "loguniform": tune.loguniform(1e-4, 1e-2),
+            "qloguniform": tune.qloguniform(1e-4, 1e-1, 5e-4),
+            "choice": tune.choice([2, 3, 4]),
+            "randint": tune.randint(-9, 15),
+            "qrandint": tune.qrandint(-21, 12, 3),
+            "randn": tune.randn(10, 2),
+            "qrandn": tune.qrandn(10, 2, 0.2),
+        }
+        for _, (_, generated) in zip(
+                range(10), generate_variants({
+                    "config": config
+                })):
+            out = generated["config"]
+
+            self.assertAlmostEqual(out["func"], out["uniform"] * 0.01)
+
+            self.assertGreater(out["uniform"], -5)
+            self.assertLess(out["uniform"], -1)
+
+            self.assertGreater(out["quniform"], 3.2)
+            self.assertLessEqual(out["quniform"], 5.4)
+            self.assertAlmostEqual(out["quniform"] / 0.2,
+                                   round(out["quniform"] / 0.2))
+
+            self.assertGreater(out["loguniform"], 1e-4)
+            self.assertLess(out["loguniform"], 1e-2)
+
+            self.assertGreater(out["qloguniform"], 1e-4)
+            self.assertLessEqual(out["qloguniform"], 1e-1)
+            self.assertAlmostEqual(out["qloguniform"] / 5e-4,
+                                   round(out["qloguniform"] / 5e-4))
+
+            self.assertIn(out["choice"], [2, 3, 4])
+
+            self.assertGreater(out["randint"], -9)
+            self.assertLess(out["randint"], 15)
+
+            self.assertGreater(out["qrandint"], -21)
+            self.assertLessEqual(out["qrandint"], 12)
+            self.assertEqual(out["qrandint"] % 3, 0)
+
+            # Very improbable
+            self.assertGreater(out["randn"], 0)
+            self.assertLess(out["randn"], 20)
+
+            self.assertGreater(out["qrandn"], 0)
+            self.assertLess(out["qrandn"], 20)
+            self.assertAlmostEqual(out["qrandn"] / 0.2,
+                                   round(out["qrandn"] / 0.2))
+
     def testBoundedFloat(self):
         bounded = tune.sample.Float(-4.2, 8.3)
 

From d9cb33ff489903441e633e21a37f65fe6b1d0500 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 13:17:28 +0100
Subject: [PATCH 27/48] Add to API docs

---
 doc/source/conf.py                       |  1 +
 doc/source/tune/api_docs/grid_random.rst | 15 +++++++++++++++
 python/ray/tune/sample.py                | 23 +++--------------------
 python/ray/tune/tests/test_sample.py     | 17 -----------------
 4 files changed, 19 insertions(+), 37 deletions(-)

diff --git a/doc/source/conf.py b/doc/source/conf.py
index cdcb293f858f..e9a701f30e76 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -23,6 +23,7 @@
 # These lines added to enable Sphinx to work without installing Ray.
 import mock
 MOCK_MODULES = [
+    "ax",
     "blist",
     "gym",
     "gym.spaces",
diff --git a/doc/source/tune/api_docs/grid_random.rst b/doc/source/tune/api_docs/grid_random.rst
index 5f008fc8d279..a69869f08b5b 100644
--- a/doc/source/tune/api_docs/grid_random.rst
+++ b/doc/source/tune/api_docs/grid_random.rst
@@ -164,16 +164,31 @@ tune.randn
 
 .. autofunction:: ray.tune.randn
 
+tune.qrandn
+~~~~~~~~~~~
+
+.. autofunction:: ray.tune.qrandn
+
 tune.loguniform
 ~~~~~~~~~~~~~~~
 
 .. autofunction:: ray.tune.loguniform
 
+tune.qloguniform
+~~~~~~~~~~~~~~~~
+
+.. autofunction:: ray.tune.qloguniform
+
 tune.uniform
 ~~~~~~~~~~~~
 
 .. autofunction:: ray.tune.uniform
 
+tune.quniform
+~~~~~~~~~~~~~
+
+.. autofunction:: ray.tune.quniform
+
 tune.choice
 ~~~~~~~~~~~
 
diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index 9ab7ee7b9850..e0d670777e44 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -2,9 +2,7 @@
 import random
 from copy import copy
 from numbers import Number
-from typing import Any, Callable, Dict, Iterator, List, Optional, \
-    Sequence, \
-    Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
 import numpy as np
 
@@ -241,21 +239,6 @@ def __getitem__(self, item):
         return self.categories[item]
 
 
-class Iterative(Domain):
-    class _NextSampler(BaseSampler):
-        def sample(self,
-                   domain: "Iterative",
-                   spec: Optional[Union[List[Dict], Dict]] = None,
-                   size: int = 1):
-            items = [next(domain.iterator) for _ in range(size)]
-            return items if len(items) > 1 else domain.cast(items[0])
-
-    default_sampler_cls = _NextSampler
-
-    def __init__(self, iterator: Iterator):
-        self.iterator = iterator
-
-
 class Function(Domain):
     class _CallSampler(BaseSampler):
         def sample(self,
@@ -326,7 +309,7 @@ def quniform(min: float, max: float, q: float):
     ``np.random.uniform(1, 10))``
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
-        Quantization makes the upper bound inclusive.
+    Quantization makes the upper bound inclusive.
 
     """
     return Float(min, max).uniform().quantized(q)
@@ -344,7 +327,7 @@ def loguniform(min: float, max: float, base: float = 10):
     return Float(min, max).loguniform(base)
 
 
-def qloguniform(min: float, max: float, q, base=10):
+def qloguniform(min: float, max: float, q: float, base: float = 10):
     """Sugar for sampling in different orders of magnitude.
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index c55953d38ce5..0ad14760539c 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -118,23 +118,6 @@ def testCategorical(self):
         self.assertTrue(any([-2 <= s <= 2 for s in samples]))
         self.assertTrue(all([c in samples for c in categories]))
 
-    def testIterative(self):
-        categories = [-2, -1, 0, 1, 2]
-
-        def test_iter():
-            for i in categories:
-                yield i
-
-        itr = tune.sample.Iterative(test_iter())
-        samples = itr.sample(size=5)
-        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
-        self.assertTrue(all([c in samples for c in categories]))
-
-        itr = tune.sample.Iterative(iter(categories))
-        samples = itr.sample(size=5)
-        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
-        self.assertTrue(all([c in samples for c in categories]))
-
     def testFunction(self):
         def sample(spec):
             return np.random.uniform(-4, 4)

From 98623f346183d7115a011eaa57b190793ef1987d Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 13:21:03 +0100
Subject: [PATCH 28/48] Fix docs

---
 doc/source/conf.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/source/conf.py b/doc/source/conf.py
index 66f21417ee4e..e6123003e199 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -32,6 +32,7 @@ def __getattr__(cls, name):
 
 MOCK_MODULES = [
     "ax",
+    "ax.service.ax_client",
     "blist",
     "gym",
     "gym.spaces",

From 65f6c9ad340a22d7d8d36022c53ce05970d5d900 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 13:44:00 +0100
Subject: [PATCH 29/48] Update metric and mode only when set. Set default
 metric and mode to experiment analysis object.

---
 .../ray/tune/analysis/experiment_analysis.py  | 109 +++++++++++++-----
 python/ray/tune/suggest/ax.py                 |   6 +-
 python/ray/tune/suggest/bayesopt.py           |  10 +-
 python/ray/tune/suggest/hyperopt.py           |  10 +-
 python/ray/tune/suggest/optuna.py             |   6 +-
 .../tune/tests/test_experiment_analysis.py    |  32 ++---
 python/ray/tune/tune.py                       |  10 +-
 7 files changed, 127 insertions(+), 56 deletions(-)

diff --git a/python/ray/tune/analysis/experiment_analysis.py b/python/ray/tune/analysis/experiment_analysis.py
index 850b3fcb349d..22b90455d1ec 100644
--- a/python/ray/tune/analysis/experiment_analysis.py
+++ b/python/ray/tune/analysis/experiment_analysis.py
@@ -22,7 +22,7 @@ class Analysis:
     To use this class, the experiment must be executed with the JsonLogger.
     """
 
-    def __init__(self, experiment_dir):
+    def __init__(self, experiment_dir, default_metric=None, default_mode=None):
         experiment_dir = os.path.expanduser(experiment_dir)
         if not os.path.isdir(experiment_dir):
             raise ValueError(
@@ -31,6 +31,9 @@ def __init__(self, experiment_dir):
         self._configs = {}
         self._trial_dataframes = {}
 
+        self.default_metric = default_metric
+        self.default_mode = default_mode
+
         if not pd:
             logger.warning(
                 "pandas not installed. Run `pip install pandas` for "
@@ -57,13 +60,18 @@ def dataframe(self, metric=None, mode=None):
                 rows[path].update(logdir=path)
         return pd.DataFrame(list(rows.values()))
 
-    def get_best_config(self, metric, mode="max"):
+    def get_best_config(self, metric=None, mode=None):
         """Retrieve the best config corresponding to the trial.
 
         Args:
-            metric (str): Key for trial info to order on.
-            mode (str): One of [min, max].
+            metric (str): Key for trial info to order on. Defaults to
+                ``self.default_metric``.
+            mode (str): One of [min, max]. Defaults to
+                ``self.default_mode``.
         """
+        metric = metric or self.default_metric
+        mode = mode or self.default_mode
+
         rows = self._retrieve_rows(metric=metric, mode=mode)
         if not rows:
             # only nans encountered when retrieving rows
@@ -77,13 +85,17 @@ def get_best_config(self, metric, mode="max"):
         best_path = compare_op(rows, key=lambda k: rows[k][metric])
         return all_configs[best_path]
 
-    def get_best_logdir(self, metric, mode="max"):
+    def get_best_logdir(self, metric=None, mode=None):
         """Retrieve the logdir corresponding to the best trial.
 
         Args:
-            metric (str): Key for trial info to order on.
-            mode (str): One of [min, max].
+            metric (str): Key for trial info to order on. Defaults to
+                ``self.default_metric``.
+            mode (str): One of [min, max]. Defaults to ``self.default_mode``.
         """
+        metric = metric or self.default_metric
+        mode = mode or self.default_mode
+
         assert mode in ["max", "min"]
         df = self.dataframe(metric=metric, mode=mode)
         mode_idx = pd.Series.idxmax if mode == "max" else pd.Series.idxmin
@@ -140,17 +152,20 @@ def get_all_configs(self, prefix=False):
                 "Couldn't read config from {} paths".format(fail_count))
         return self._configs
 
-    def get_trial_checkpoints_paths(self, trial, metric=TRAINING_ITERATION):
+    def get_trial_checkpoints_paths(self, trial, metric=None):
         """Gets paths and metrics of all persistent checkpoints of a trial.
 
         Args:
             trial (Trial): The log directory of a trial, or a trial instance.
             metric (str): key for trial info to return, e.g. "mean_accuracy".
-                "training_iteration" is used by default.
+                "training_iteration" is used by default if no value was
+                passed to ``self.default_metric``.
 
         Returns:
             List of [path, metric] for all persistent checkpoints of the trial.
         """
+        metric = metric or self.default_metric or TRAINING_ITERATION
+
         if isinstance(trial, str):
             trial_dir = os.path.expanduser(trial)
             # Get checkpoints from logdir.
@@ -167,19 +182,22 @@ def get_trial_checkpoints_paths(self, trial, metric=TRAINING_ITERATION):
         else:
             raise ValueError("trial should be a string or a Trial instance.")
 
-    def get_best_checkpoint(self, trial, metric=TRAINING_ITERATION,
-                            mode="max"):
+    def get_best_checkpoint(self, trial, metric=None, mode="max"):
         """Gets best persistent checkpoint path of provided trial.
 
         Args:
             trial (Trial): The log directory of a trial, or a trial instance.
             metric (str): key of trial info to return, e.g. "mean_accuracy".
-                "training_iteration" is used by default.
-            mode (str): Either "min" or "max".
+                "training_iteration" is used by default if no value was
+                passed to ``self.default_metric``.
+            mode (str): One of [min, max]. Defaults to ``self.default_mode``.
 
         Returns:
             Path for best checkpoint of trial determined by metric
         """
+        metric = metric or self.default_metric or TRAINING_ITERATION
+        mode = mode or self.default_mode
+
         assert mode in ["max", "min"]
         checkpoint_paths = self.get_trial_checkpoints_paths(trial, metric)
         if mode == "max":
@@ -235,6 +253,10 @@ class ExperimentAnalysis(Analysis):
             Experiment.local_dir/Experiment.name/experiment_state.json
         trials (list|None): List of trials that can be accessed via
             `analysis.trials`.
+        default_metric (str): Default metric for `get_best_trial()` and its
+            derivatives.
+        default_mode (str): Default mode for `get_best_trial()` and its
+            derivatives.
 
     Example:
         >>> tune.run(my_trainable, name="my_exp", local_dir="~/tune_results")
@@ -242,7 +264,11 @@ class ExperimentAnalysis(Analysis):
         >>>     experiment_checkpoint_path="~/tune_results/my_exp/state.json")
     """
 
-    def __init__(self, experiment_checkpoint_path, trials=None):
+    def __init__(self,
+                 experiment_checkpoint_path,
+                 trials=None,
+                 default_metric=None,
+                 default_mode=None):
         experiment_checkpoint_path = os.path.expanduser(
             experiment_checkpoint_path)
         if not os.path.isfile(experiment_checkpoint_path):
@@ -256,17 +282,24 @@ def __init__(self, experiment_checkpoint_path, trials=None):
             raise TuneError("Experiment state invalid; no checkpoints found.")
         self._checkpoints = _experiment_state["checkpoints"]
         self.trials = trials
+
         super(ExperimentAnalysis, self).__init__(
-            os.path.dirname(experiment_checkpoint_path))
+            os.path.dirname(experiment_checkpoint_path), default_metric,
+            default_mode)
 
-    def get_best_trial(self, metric, mode="max", scope="all"):
+    def get_best_trial(self, metric=None, mode=None, scope="all"):
         """Retrieve the best trial object.
 
-        Compares all trials' scores on `metric`.
+        Compares all trials' scores on ``metric``.
+        If ``metric`` is not specified, ``self.default_metric`` will be used.
+        If `mode` is not specified, ``self.default_mode`` will be used.
+        These values are usually initialized by passing the ``metric`` and
+        ``mode`` parameters to ``tune.run()``.
 
         Args:
-            metric (str): Key for trial info to order on.
-            mode (str): One of [min, max].
+            metric (str): Key for trial info to order on. Defaults to
+                ``self.default_metric``.
+            mode (str): One of [min, max]. Defaults to ``self.default_mode``.
             scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
                 If `scope=last`, only look at each trial's final step for
                 `metric`, and compare across trials based on `mode=[min,max]`.
@@ -278,16 +311,24 @@ def get_best_trial(self, metric, mode="max", scope="all"):
                 If `scope=all`, find each trial's min/max score for `metric`
                 based on `mode`, and compare trials based on `mode=[min,max]`.
         """
+        metric = metric or self.default_metric
+        mode = mode or self.default_mode
+
         if mode not in ["max", "min"]:
             raise ValueError(
                 "ExperimentAnalysis: attempting to get best trial for "
-                "metric {} for mode {} not in [\"max\", \"min\"]".format(
+                "metric {} for mode {} not in [\"max\", \"min\"]. "
+                "If you didn't pass a `mode` parameter to `tune.run()`, "
+                "you have to pass one when fetching the best trial.".format(
                     metric, mode))
         if scope not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]:
             raise ValueError(
                 "ExperimentAnalysis: attempting to get best trial for "
                 "metric {} for scope {} not in [\"all\", \"last\", \"avg\", "
-                "\"last-5-avg\", \"last-10-avg\"]".format(metric, scope))
+                "\"last-5-avg\", \"last-10-avg\"]. "
+                "If you didn't pass a `metric` parameter to `tune.run()`, "
+                "you have to pass one when fetching the best trial.".format(
+                    metric, scope))
         best_trial = None
         best_metric_score = None
         for trial in self.trials:
@@ -311,16 +352,25 @@ def get_best_trial(self, metric, mode="max", scope="all"):
                 best_metric_score = metric_score
                 best_trial = trial
 
+        if not best_trial:
+            logger.warning(
+                "Could not find best trial. Did you pass the correct `metric`"
+                "parameter?")
         return best_trial
 
-    def get_best_config(self, metric, mode="max", scope="all"):
+    def get_best_config(self, metric=None, mode=None, scope="all"):
         """Retrieve the best config corresponding to the trial.
 
         Compares all trials' scores on `metric`.
+        If ``metric`` is not specified, ``self.default_metric`` will be used.
+        If `mode` is not specified, ``self.default_mode`` will be used.
+        These values are usually initialized by passing the ``metric`` and
+        ``mode`` parameters to ``tune.run()``.
 
         Args:
-            metric (str): Key for trial info to order on.
-            mode (str): One of [min, max].
+            metric (str): Key for trial info to order on. Defaults to
+                ``self.default_metric``.
+            mode (str): One of [min, max]. Defaults to ``self.default_mode``.
             scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
                 If `scope=last`, only look at each trial's final step for
                 `metric`, and compare across trials based on `mode=[min,max]`.
@@ -335,14 +385,19 @@ def get_best_config(self, metric, mode="max", scope="all"):
         best_trial = self.get_best_trial(metric, mode, scope)
         return best_trial.config if best_trial else None
 
-    def get_best_logdir(self, metric, mode="max", scope="all"):
+    def get_best_logdir(self, metric=None, mode=None, scope="all"):
         """Retrieve the logdir corresponding to the best trial.
 
         Compares all trials' scores on `metric`.
+        If ``metric`` is not specified, ``self.default_metric`` will be used.
+        If `mode` is not specified, ``self.default_mode`` will be used.
+        These values are usually initialized by passing the ``metric`` and
+        ``mode`` parameters to ``tune.run()``.
 
         Args:
-            metric (str): Key for trial info to order on.
-            mode (str): One of [min, max].
+            metric (str): Key for trial info to order on. Defaults to
+                ``self.default_metric``.
+            mode (str): One of [min, max]. Defaults to ``self.default_mode``.
             scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
                 If `scope=last`, only look at each trial's final step for
                 `metric`, and compare across trials based on `mode=[min,max]`.
diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index cb23a4c581b5..09c9ef20418f 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -158,8 +158,10 @@ def set_search_properties(self, metric, mode, config):
             return False
         space = self.convert_search_space(config)
         self._space = space
-        self._metric = metric
-        self._mode = mode
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
         self.setup_experiment()
         return True
 
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index 9248c87b441c..e8b5063b6c64 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -185,12 +185,14 @@ def set_search_properties(self, metric, mode, config):
             return False
         space = self.convert_search_space(config)
         self._space = space
-        self._metric = metric
-        self._mode = mode
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
 
-        if mode == "max":
+        if self._mode == "max":
             self._metric_op = 1.
-        elif mode == "min":
+        elif self._mode == "min":
             self._metric_op = -1.
 
         self.setup_optimizer()
diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 2f94591cc6b7..9b28add05e71 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -149,12 +149,14 @@ def set_search_properties(self, metric, mode, config):
         space = self.convert_search_space(config)
         self.domain = hpo.Domain(lambda spc: spc, space)
 
-        self._metric = metric
-        self._mode = mode
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
 
-        if mode == "max":
+        if self._mode == "max":
             self.metric_op = -1.
-        elif mode == "min":
+        elif self._mode == "min":
             self.metric_op = 1.
 
         return True
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index db1d02590367..7265069cfc33 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -132,8 +132,10 @@ def set_search_properties(self, metric, mode, config):
             return False
         space = self.convert_search_space(config)
         self._space = space
-        self._metric = metric
-        self._mode = mode
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
         self.setup_study(mode)
         self._config = config
         return True
diff --git a/python/ray/tune/tests/test_experiment_analysis.py b/python/ray/tune/tests/test_experiment_analysis.py
index 5b1f4c7bb22c..790853901627 100644
--- a/python/ray/tune/tests/test_experiment_analysis.py
+++ b/python/ray/tune/tests/test_experiment_analysis.py
@@ -29,6 +29,8 @@ def run_test_exp(self):
         self.ea = run(
             MyTrainableClass,
             name=self.test_name,
+            metric=self.metric,
+            mode="max",
             local_dir=self.test_dir,
             stop={"training_iteration": 1},
             checkpoint_freq=1,
@@ -43,6 +45,8 @@ def nan_test_exp(self):
         nan_ea = run(
             lambda x: nan,
             name="testing_nan",
+            metric=self.metric,
+            mode="max",
             local_dir=self.test_dir,
             stop={"training_iteration": 1},
             checkpoint_freq=1,
@@ -73,61 +77,61 @@ def testTrialDataframe(self):
         self.assertEqual(trial_df.shape[0], 1)
 
     def testBestConfig(self):
-        best_config = self.ea.get_best_config(self.metric)
+        best_config = self.ea.get_best_config()
         self.assertTrue(isinstance(best_config, dict))
         self.assertTrue("width" in best_config)
         self.assertTrue("height" in best_config)
 
     def testBestConfigNan(self):
         nan_ea = self.nan_test_exp()
-        best_config = nan_ea.get_best_config(self.metric)
+        best_config = nan_ea.get_best_config()
         self.assertIsNone(best_config)
 
     def testBestLogdir(self):
-        logdir = self.ea.get_best_logdir(self.metric)
+        logdir = self.ea.get_best_logdir()
         self.assertTrue(logdir.startswith(self.test_path))
-        logdir2 = self.ea.get_best_logdir(self.metric, mode="min")
+        logdir2 = self.ea.get_best_logdir(mode="min")
         self.assertTrue(logdir2.startswith(self.test_path))
         self.assertNotEquals(logdir, logdir2)
 
     def testBestLogdirNan(self):
         nan_ea = self.nan_test_exp()
-        logdir = nan_ea.get_best_logdir(self.metric)
+        logdir = nan_ea.get_best_logdir()
         self.assertIsNone(logdir)
 
     def testGetTrialCheckpointsPathsByTrial(self):
-        best_trial = self.ea.get_best_trial(self.metric)
+        best_trial = self.ea.get_best_trial()
         checkpoints_metrics = self.ea.get_trial_checkpoints_paths(best_trial)
-        logdir = self.ea.get_best_logdir(self.metric)
+        logdir = self.ea.get_best_logdir()
         expected_path = os.path.join(logdir, "checkpoint_1", "checkpoint")
         assert checkpoints_metrics[0][0] == expected_path
         assert checkpoints_metrics[0][1] == 1
 
     def testGetTrialCheckpointsPathsByPath(self):
-        logdir = self.ea.get_best_logdir(self.metric)
+        logdir = self.ea.get_best_logdir()
         checkpoints_metrics = self.ea.get_trial_checkpoints_paths(logdir)
         expected_path = os.path.join(logdir, "checkpoint_1/", "checkpoint")
         assert checkpoints_metrics[0][0] == expected_path
         assert checkpoints_metrics[0][1] == 1
 
     def testGetTrialCheckpointsPathsWithMetricByTrial(self):
-        best_trial = self.ea.get_best_trial(self.metric)
-        paths = self.ea.get_trial_checkpoints_paths(best_trial, self.metric)
-        logdir = self.ea.get_best_logdir(self.metric)
+        best_trial = self.ea.get_best_trial()
+        paths = self.ea.get_trial_checkpoints_paths(best_trial)
+        logdir = self.ea.get_best_logdir()
         expected_path = os.path.join(logdir, "checkpoint_1", "checkpoint")
         assert paths[0][0] == expected_path
         assert paths[0][1] == best_trial.metric_analysis[self.metric]["last"]
 
     def testGetTrialCheckpointsPathsWithMetricByPath(self):
-        best_trial = self.ea.get_best_trial(self.metric)
-        logdir = self.ea.get_best_logdir(self.metric)
+        best_trial = self.ea.get_best_trial()
+        logdir = self.ea.get_best_logdir()
         paths = self.ea.get_trial_checkpoints_paths(best_trial, self.metric)
         expected_path = os.path.join(logdir, "checkpoint_1", "checkpoint")
         assert paths[0][0] == expected_path
         assert paths[0][1] == best_trial.metric_analysis[self.metric]["last"]
 
     def testGetBestCheckpoint(self):
-        best_trial = self.ea.get_best_trial(self.metric)
+        best_trial = self.ea.get_best_trial()
         checkpoints_metrics = self.ea.get_trial_checkpoints_paths(best_trial)
         expected_path = max(checkpoints_metrics, key=lambda x: x[1])[0]
         best_checkpoint = self.ea.get_best_checkpoint(best_trial, self.metric)
diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py
index b5800d73408d..009092d440c8 100644
--- a/python/ray/tune/tune.py
+++ b/python/ray/tune/tune.py
@@ -67,8 +67,8 @@ def _report_progress(runner, reporter, done=False):
 
 def run(run_or_experiment,
         name=None,
-        metric="episode_reward_mean",
-        mode="max",
+        metric=None,
+        mode=None,
         stop=None,
         config=None,
         resources_per_trial=None,
@@ -415,7 +415,11 @@ def run(run_or_experiment,
     trials = runner.get_trials()
     if return_trials:
         return trials
-    return ExperimentAnalysis(runner.checkpoint_file, trials=trials)
+    return ExperimentAnalysis(
+        runner.checkpoint_file,
+        trials=trials,
+        default_metric=metric,
+        default_mode=mode)
 
 
 def run_experiments(experiments,

From 04cc380c5e24758659a3d485db7deb34419ad28f Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 13:53:56 +0100
Subject: [PATCH 30/48] Fix experiment analysis tests

---
 python/ray/tune/tests/test_experiment_analysis.py     | 7 +++++--
 python/ray/tune/tests/test_experiment_analysis_mem.py | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/python/ray/tune/tests/test_experiment_analysis.py b/python/ray/tune/tests/test_experiment_analysis.py
index 790853901627..a2833318b46b 100644
--- a/python/ray/tune/tests/test_experiment_analysis.py
+++ b/python/ray/tune/tests/test_experiment_analysis.py
@@ -9,6 +9,7 @@
 import ray
 from ray.tune import run, sample_from
 from ray.tune.examples.async_hyperband_example import MyTrainableClass
+from ray.tune.result import TRAINING_ITERATION
 
 
 class ExperimentAnalysisSuite(unittest.TestCase):
@@ -101,7 +102,8 @@ def testBestLogdirNan(self):
 
     def testGetTrialCheckpointsPathsByTrial(self):
         best_trial = self.ea.get_best_trial()
-        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(best_trial)
+        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(
+            best_trial, TRAINING_ITERATION)
         logdir = self.ea.get_best_logdir()
         expected_path = os.path.join(logdir, "checkpoint_1", "checkpoint")
         assert checkpoints_metrics[0][0] == expected_path
@@ -109,7 +111,8 @@ def testGetTrialCheckpointsPathsByTrial(self):
 
     def testGetTrialCheckpointsPathsByPath(self):
         logdir = self.ea.get_best_logdir()
-        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(logdir)
+        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(
+            logdir, TRAINING_ITERATION)
         expected_path = os.path.join(logdir, "checkpoint_1/", "checkpoint")
         assert checkpoints_metrics[0][0] == expected_path
         assert checkpoints_metrics[0][1] == 1
diff --git a/python/ray/tune/tests/test_experiment_analysis_mem.py b/python/ray/tune/tests/test_experiment_analysis_mem.py
index 94e544b766ec..65e5320251c6 100644
--- a/python/ray/tune/tests/test_experiment_analysis_mem.py
+++ b/python/ray/tune/tests/test_experiment_analysis_mem.py
@@ -156,7 +156,7 @@ def testDataframe(self):
 
     def testBestLogdir(self):
         analysis = Analysis(self.test_dir)
-        logdir = analysis.get_best_logdir(self.metric)
+        logdir = analysis.get_best_logdir(self.metric, mode="max")
         self.assertTrue(logdir.startswith(self.test_dir))
         logdir2 = analysis.get_best_logdir(self.metric, mode="min")
         self.assertTrue(logdir2.startswith(self.test_dir))

From e9210392a3f4d04b790269a2ad7a24e03f6daab9 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 14:31:50 +0100
Subject: [PATCH 31/48] Raise error when delimiter is used in the config keys

---
 python/ray/tune/suggest/ax.py        |  2 +-
 python/ray/tune/suggest/bayesopt.py  |  3 ++-
 python/ray/tune/suggest/optuna.py    |  2 +-
 python/ray/tune/tests/test_sample.py |  8 ++++++++
 python/ray/tune/utils/util.py        | 13 ++++++++++++-
 5 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 09c9ef20418f..e51e5eb8c297 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -204,7 +204,7 @@ def _process_result(self, trial_id, result):
 
     @staticmethod
     def convert_search_space(spec: Dict):
-        spec = flatten_dict(spec)
+        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index e8b5063b6c64..c88a701464e0 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -328,7 +328,8 @@ def restore(self, checkpoint_path):
 
     @staticmethod
     def convert_search_space(spec: Dict):
-        spec = flatten_dict(spec)
+        print(spec)
+        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 7265069cfc33..7dab0152b6f2 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -189,7 +189,7 @@ def restore(self, checkpoint_path):
 
     @staticmethod
     def convert_search_space(spec: Dict):
-        spec = flatten_dict(spec)
+        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if not domain_vars and not grid_vars:
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 0ad14760539c..b7423b0bcdd8 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -233,6 +233,14 @@ def testConvertBayesOpt(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
+        searcher = BayesOptSearch()
+        invalid_config = {"a/b": tune.uniform(4.0, 8.0)}
+        with self.assertRaises(ValueError):
+            searcher.set_search_properties("none", "max", invalid_config)
+        invalid_config = {"a": {"b/c": tune.uniform(4.0, 8.0)}}
+        with self.assertRaises(ValueError):
+            searcher.set_search_properties("none", "max", invalid_config)
+
         searcher = BayesOptSearch()
         analysis = tune.run(
             _mock_objective,
diff --git a/python/ray/tune/utils/util.py b/python/ray/tune/utils/util.py
index 8b8136657953..ff5e7b15dc79 100644
--- a/python/ray/tune/utils/util.py
+++ b/python/ray/tune/utils/util.py
@@ -215,14 +215,25 @@ def deep_update(original,
     return original
 
 
-def flatten_dict(dt, delimiter="/"):
+def flatten_dict(dt, delimiter="/", prevent_delimiter=False):
     dt = copy.deepcopy(dt)
+    if prevent_delimiter and any(delimiter in key for key in dt):
+        # Raise if delimiter is any of the keys
+        raise ValueError(
+            "Found delimiter `{}` in key when trying to flatten array."
+            "Please avoid using the delimiter in your specification.")
     while any(isinstance(v, dict) for v in dt.values()):
         remove = []
         add = {}
         for key, value in dt.items():
             if isinstance(value, dict):
                 for subkey, v in value.items():
+                    if prevent_delimiter and delimiter in subkey:
+                        # Raise  if delimiter is in any of the subkeys
+                        raise ValueError(
+                            "Found delimiter `{}` in key when trying to "
+                            "flatten array. Please avoid using the delimiter "
+                            "in your specification.")
                     add[delimiter.join([key, subkey])] = v
                 remove.append(key)
         dt.update(add)

From 7940dbcaa71262f6e1fd41412a07c0da8ee9cb50 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 16:50:11 +0100
Subject: [PATCH 32/48] Added randint/qrandint to API docs, added additional
 check in tune.run

---
 doc/source/tune/api_docs/grid_random.rst | 10 ++++++++++
 python/ray/tune/suggest/suggestion.py    |  2 +-
 python/ray/tune/tune.py                  |  3 +++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/doc/source/tune/api_docs/grid_random.rst b/doc/source/tune/api_docs/grid_random.rst
index a69869f08b5b..32347eba23c4 100644
--- a/doc/source/tune/api_docs/grid_random.rst
+++ b/doc/source/tune/api_docs/grid_random.rst
@@ -189,6 +189,16 @@ tune.quniform
 
 .. autofunction:: ray.tune.quniform
 
+tune.randint
+~~~~~~~~~~~~
+
+.. autofunction:: ray.tune.randint
+
+tune.qrandint
+~~~~~~~~~~~~~
+
+.. autofunction:: ray.tune.qrandint
+
 tune.choice
 ~~~~~~~~~~~
 
diff --git a/python/ray/tune/suggest/suggestion.py b/python/ray/tune/suggest/suggestion.py
index 95e5bea8efff..b4b0e81839e4 100644
--- a/python/ray/tune/suggest/suggestion.py
+++ b/python/ray/tune/suggest/suggestion.py
@@ -83,7 +83,7 @@ def set_search_properties(self, metric, mode, config):
             mode (str): One of ["min", "max"]. Direction to optimize.
             config (dict): Tune config dict.
         """
-        return True
+        return False
 
     def on_trial_result(self, trial_id, result):
         """Optional notification for result during training.
diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py
index 009092d440c8..329f29a1e3c3 100644
--- a/python/ray/tune/tune.py
+++ b/python/ray/tune/tune.py
@@ -332,6 +332,9 @@ def run(run_or_experiment,
     if not search_alg:
         search_alg = BasicVariantGenerator()
 
+    if mode:
+        assert mode in ["min", "max"], "`mode` must be one of [min, max]"
+
     if config and not search_alg.set_search_properties(metric, mode, config):
         logger.warning(
             "You passed a `config` parameter to `tune.run()`, but the "

From 8a8b880676fcf00f7d26ba535f9c9f8106ff47ff Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 21:31:33 +0100
Subject: [PATCH 33/48] Fix tests

---
 python/ray/tune/examples/mnist_pytorch.py                   | 3 ++-
 python/ray/tune/examples/mnist_pytorch_trainable.py         | 3 ++-
 python/ray/tune/examples/pbt_convnet_example.py             | 5 +++--
 python/ray/tune/examples/pbt_convnet_function_example.py    | 4 ++--
 python/ray/tune/schedulers/pbt.py                           | 4 ++--
 python/ray/tune/tests/example.py                            | 3 ++-
 python/ray/tune/tests/test_trial_scheduler.py               | 2 +-
 python/ray/util/sgd/tf/examples/tensorflow_train_example.py | 4 ++--
 8 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/python/ray/tune/examples/mnist_pytorch.py b/python/ray/tune/examples/mnist_pytorch.py
index 9c8f7f0c69e1..5a2c3677079c 100644
--- a/python/ray/tune/examples/mnist_pytorch.py
+++ b/python/ray/tune/examples/mnist_pytorch.py
@@ -141,4 +141,5 @@ def train_mnist(config):
             "use_gpu": int(args.cuda)
         })
 
-    print("Best config is:", analysis.get_best_config(metric="mean_accuracy"))
+    print("Best config is:",
+          analysis.get_best_config(metric="mean_accuracy", mode="max"))
diff --git a/python/ray/tune/examples/mnist_pytorch_trainable.py b/python/ray/tune/examples/mnist_pytorch_trainable.py
index 956a9c024108..c623111daf83 100644
--- a/python/ray/tune/examples/mnist_pytorch_trainable.py
+++ b/python/ray/tune/examples/mnist_pytorch_trainable.py
@@ -86,4 +86,5 @@ def load_checkpoint(self, checkpoint_path):
             "momentum": tune.uniform(0.1, 0.9),
         })
 
-    print("Best config is:", analysis.get_best_config(metric="mean_accuracy"))
+    print("Best config is:",
+          analysis.get_best_config(metric="mean_accuracy", mode="max"))
diff --git a/python/ray/tune/examples/pbt_convnet_example.py b/python/ray/tune/examples/pbt_convnet_example.py
index c9f455ccc0fe..d996d0edda6c 100644
--- a/python/ray/tune/examples/pbt_convnet_example.py
+++ b/python/ray/tune/examples/pbt_convnet_example.py
@@ -131,8 +131,9 @@ def stop_all(self):
         })
     # __tune_end__
 
-    best_trial = analysis.get_best_trial("mean_accuracy")
-    best_checkpoint = analysis.get_best_checkpoint(best_trial, metric="mean_accuracy")
+    best_trial = analysis.get_best_trial("mean_accuracy", "max")
+    best_checkpoint = analysis.get_best_checkpoint(
+        best_trial, metric="mean_accuracy", mode="max")
     restored_trainable = PytorchTrainable()
     restored_trainable.restore(best_checkpoint)
     best_model = restored_trainable.model
diff --git a/python/ray/tune/examples/pbt_convnet_function_example.py b/python/ray/tune/examples/pbt_convnet_function_example.py
index 758148a29fba..a607bed1280a 100644
--- a/python/ray/tune/examples/pbt_convnet_function_example.py
+++ b/python/ray/tune/examples/pbt_convnet_function_example.py
@@ -116,9 +116,9 @@ def stop_all(self):
         })
     # __tune_end__
 
-    best_trial = analysis.get_best_trial("mean_accuracy")
+    best_trial = analysis.get_best_trial("mean_accuracy", mode="max")
     best_checkpoint_path = analysis.get_best_checkpoint(
-        best_trial, metric="mean_accuracy")
+        best_trial, metric="mean_accuracy", mode="max")
     best_model = ConvNet()
     best_checkpoint = torch.load(
         os.path.join(best_checkpoint_path, "checkpoint"))
diff --git a/python/ray/tune/schedulers/pbt.py b/python/ray/tune/schedulers/pbt.py
index 40e32e22e387..70137e8de3f6 100644
--- a/python/ray/tune/schedulers/pbt.py
+++ b/python/ray/tune/schedulers/pbt.py
@@ -9,7 +9,7 @@
 from ray.tune.error import TuneError
 from ray.tune.result import TRAINING_ITERATION
 from ray.tune.logger import _SafeFallbackEncoder
-from ray.tune.sample import Domain, sample_from
+from ray.tune.sample import Domain, Function
 from ray.tune.schedulers import FIFOScheduler, TrialScheduler
 from ray.tune.suggest.variant_generator import format_vars
 from ray.tune.trial import Trial, Checkpoint
@@ -232,7 +232,7 @@ def __init__(self,
                 raise TypeError("`hyperparam_mutation` values must be either "
                                 "a List, Dict, a tune search space object, or "
                                 "a callable.")
-            if type(value) is sample_from:
+            if isinstance(value, Function):
                 raise ValueError("arbitrary tune.sample_from objects are not "
                                  "supported for `hyperparam_mutation` values."
                                  "You must use other built in primitives like"
diff --git a/python/ray/tune/tests/example.py b/python/ray/tune/tests/example.py
index be0bd2d17148..69d1f854b577 100644
--- a/python/ray/tune/tests/example.py
+++ b/python/ray/tune/tests/example.py
@@ -35,7 +35,8 @@ def training_function(config):
         "beta": tune.choice([1, 2, 3])
     })
 
-print("Best config: ", analysis.get_best_config(metric="mean_loss"))
+print("Best config: ", analysis.get_best_config(
+    metric="mean_loss", mode="min"))
 
 # Get a dataframe for analyzing trial results.
 df = analysis.dataframe()
diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py
index ec7e96c5bf14..c1c00d8acc9d 100644
--- a/python/ray/tune/tests/test_trial_scheduler.py
+++ b/python/ray/tune/tests/test_trial_scheduler.py
@@ -976,7 +976,7 @@ def testTuneSamplePrimitives(self):
     def testTuneSampleFromError(self):
         with self.assertRaises(ValueError):
             pbt, runner = self.basicSetup(hyperparam_mutations={
-                "float_factor": tune.sample_from(lambda: 100.0)
+                "float_factor": tune.sample_from(lambda _: 100.0)
             })
 
     def testPerturbationValues(self):
diff --git a/python/ray/util/sgd/tf/examples/tensorflow_train_example.py b/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
index 5982f7b8595f..8ca24f4998f3 100644
--- a/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
+++ b/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
@@ -95,8 +95,8 @@ def train_example(num_replicas=1, batch_size=128, use_gpu=False):
 
 def tune_example(num_replicas=1, use_gpu=False):
     config = {
-        "model_creator": tune.function(simple_model),
-        "data_creator": tune.function(simple_dataset),
+        "model_creator": tune.sample_from(simple_model),
+        "data_creator": tune.sample_from(simple_dataset),
         "num_replicas": num_replicas,
         "use_gpu": use_gpu,
         "trainer_config": create_config(batch_size=128)

From a4fbcf0b89643e860a6f2624bd6b58ec0e62cbba Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Tue, 1 Sep 2020 21:32:34 +0100
Subject: [PATCH 34/48] Fix linting error

---
 python/ray/tune/tests/test_sample.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index b7423b0bcdd8..81ed5703bb58 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -115,8 +115,8 @@ def testCategorical(self):
         cat = tune.sample.Categorical(categories)
 
         samples = cat.uniform().sample(size=1000)
-        self.assertTrue(any([-2 <= s <= 2 for s in samples]))
-        self.assertTrue(all([c in samples for c in categories]))
+        self.assertTrue(any(-2 <= s <= 2 for s in samples))
+        self.assertTrue(all(c in samples for c in categories))
 
     def testFunction(self):
         def sample(spec):
@@ -125,7 +125,7 @@ def sample(spec):
         fnc = tune.sample.Function(sample)
 
         samples = fnc.sample(size=1000)
-        self.assertTrue(any([-4 < s < 4 for s in samples]))
+        self.assertTrue(any(-4 < s < 4 for s in samples))
         self.assertTrue(-2 < np.mean(samples) < 2)
 
     def testQuantized(self):

From e60eb76a643bc30ef39cd16de5adb8cff649d05f Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 2 Sep 2020 10:29:46 +0100
Subject: [PATCH 35/48] Applied suggestions from code review. Re-aded
 tune.function for the time being

---
 .../ray/tune/analysis/experiment_analysis.py  | 62 +++++++++++++------
 python/ray/tune/examples/ax_example.py        |  2 +-
 .../pbt_transformers/pbt_transformers.py      |  4 +-
 python/ray/tune/sample.py                     | 36 ++++++++---
 python/ray/tune/tests/test_trial_scheduler.py |  2 +-
 python/ray/tune/tune.py                       | 14 ++---
 .../tf/examples/tensorflow_train_example.py   |  4 +-
 python/setup.py                               |  1 +
 8 files changed, 82 insertions(+), 43 deletions(-)

diff --git a/python/ray/tune/analysis/experiment_analysis.py b/python/ray/tune/analysis/experiment_analysis.py
index 22b90455d1ec..2da4c33e8883 100644
--- a/python/ray/tune/analysis/experiment_analysis.py
+++ b/python/ray/tune/analysis/experiment_analysis.py
@@ -20,6 +20,15 @@ class Analysis:
     """Analyze all results from a directory of experiments.
 
     To use this class, the experiment must be executed with the JsonLogger.
+
+    Args:
+        experiment_dir (str): Directory of the experiment to load.
+        default_metric (str): Default metric for comparing results. Can be
+            overwritten with the ``metric`` parameter in the respective
+            functions.
+        default_mode (str): Default mode for comparing results. Has to be one
+            of [min, max]. Can be overwritten with the ``mode`` parameter
+            in the respective functions.
     """
 
     def __init__(self, experiment_dir, default_metric=None, default_mode=None):
@@ -32,6 +41,9 @@ def __init__(self, experiment_dir, default_metric=None, default_mode=None):
         self._trial_dataframes = {}
 
         self.default_metric = default_metric
+        if default_mode and default_mode not in ["min", "max"]:
+            raise ValueError(
+                "`default_mode` has to be None or one of [min, max]")
         self.default_mode = default_mode
 
         if not pd:
@@ -41,6 +53,22 @@ def __init__(self, experiment_dir, default_metric=None, default_mode=None):
         else:
             self.fetch_trial_dataframes()
 
+    def _validate_metric(self, metric):
+        if not metric and not self.default_metric:
+            raise ValueError(
+                "No `metric` has been passed and  `default_metric` has "
+                "not been set. Please specify the `metric` parameter.")
+        return metric or self.default_metric
+
+    def _validate_mode(self, mode):
+        if not mode and not self.default_mode:
+            raise ValueError(
+                "No `mode` has been passed and  `default_mode` has "
+                "not been set. Please specify the `mode` parameter.")
+        if mode and mode not in ["min", "max"]:
+            raise ValueError("If set, `mode` has to be one of [min, max]")
+        return mode or self.default_mode
+
     def dataframe(self, metric=None, mode=None):
         """Returns a pandas.DataFrame object constructed from the trials.
 
@@ -69,8 +97,8 @@ def get_best_config(self, metric=None, mode=None):
             mode (str): One of [min, max]. Defaults to
                 ``self.default_mode``.
         """
-        metric = metric or self.default_metric
-        mode = mode or self.default_mode
+        metric = self._validate_metric(metric)
+        mode = self._validate_mode(mode)
 
         rows = self._retrieve_rows(metric=metric, mode=mode)
         if not rows:
@@ -93,8 +121,8 @@ def get_best_logdir(self, metric=None, mode=None):
                 ``self.default_metric``.
             mode (str): One of [min, max]. Defaults to ``self.default_mode``.
         """
-        metric = metric or self.default_metric
-        mode = mode or self.default_mode
+        metric = self._validate_metric(metric)
+        mode = self._validate_mode(mode)
 
         assert mode in ["max", "min"]
         df = self.dataframe(metric=metric, mode=mode)
@@ -182,7 +210,7 @@ def get_trial_checkpoints_paths(self, trial, metric=None):
         else:
             raise ValueError("trial should be a string or a Trial instance.")
 
-    def get_best_checkpoint(self, trial, metric=None, mode="max"):
+    def get_best_checkpoint(self, trial, metric=None, mode=None):
         """Gets best persistent checkpoint path of provided trial.
 
         Args:
@@ -196,9 +224,8 @@ def get_best_checkpoint(self, trial, metric=None, mode="max"):
             Path for best checkpoint of trial determined by metric
         """
         metric = metric or self.default_metric or TRAINING_ITERATION
-        mode = mode or self.default_mode
+        mode = self._validate_mode(mode)
 
-        assert mode in ["max", "min"]
         checkpoint_paths = self.get_trial_checkpoints_paths(trial, metric)
         if mode == "max":
             return max(checkpoint_paths, key=lambda x: x[1])[0]
@@ -253,10 +280,12 @@ class ExperimentAnalysis(Analysis):
             Experiment.local_dir/Experiment.name/experiment_state.json
         trials (list|None): List of trials that can be accessed via
             `analysis.trials`.
-        default_metric (str): Default metric for `get_best_trial()` and its
-            derivatives.
-        default_mode (str): Default mode for `get_best_trial()` and its
-            derivatives.
+        default_metric (str): Default metric for comparing results. Can be
+            overwritten with the ``metric`` parameter in the respective
+            functions.
+        default_mode (str): Default mode for comparing results. Has to be one
+            of [min, max]. Can be overwritten with the ``mode`` parameter
+            in the respective functions.
 
     Example:
         >>> tune.run(my_trainable, name="my_exp", local_dir="~/tune_results")
@@ -311,16 +340,9 @@ def get_best_trial(self, metric=None, mode=None, scope="all"):
                 If `scope=all`, find each trial's min/max score for `metric`
                 based on `mode`, and compare trials based on `mode=[min,max]`.
         """
-        metric = metric or self.default_metric
-        mode = mode or self.default_mode
+        metric = self._validate_metric(metric)
+        mode = self._validate_mode(mode)
 
-        if mode not in ["max", "min"]:
-            raise ValueError(
-                "ExperimentAnalysis: attempting to get best trial for "
-                "metric {} for mode {} not in [\"max\", \"min\"]. "
-                "If you didn't pass a `mode` parameter to `tune.run()`, "
-                "you have to pass one when fetching the best trial.".format(
-                    metric, mode))
         if scope not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]:
             raise ValueError(
                 "ExperimentAnalysis: attempting to get best trial for "
diff --git a/python/ray/tune/examples/ax_example.py b/python/ray/tune/examples/ax_example.py
index 6f5f06fb55ca..71335a094adb 100644
--- a/python/ray/tune/examples/ax_example.py
+++ b/python/ray/tune/examples/ax_example.py
@@ -106,7 +106,7 @@ def easy_objective(config):
         parameter_constraints=["x1 + x2 <= 2.0"],  # Optional.
         outcome_constraints=["l2norm <= 1.25"],  # Optional.
     )
-    algo = AxSearch(client, max_concurrent=4)
+    algo = AxSearch(ax_client=client, max_concurrent=4)
     scheduler = AsyncHyperBandScheduler(metric="hartmann6", mode="min")
     tune.run(
         easy_objective,
diff --git a/python/ray/tune/examples/pbt_transformers/pbt_transformers.py b/python/ray/tune/examples/pbt_transformers/pbt_transformers.py
index ee144f6e3983..dc1a73e5b1e1 100644
--- a/python/ray/tune/examples/pbt_transformers/pbt_transformers.py
+++ b/python/ray/tune/examples/pbt_transformers/pbt_transformers.py
@@ -155,8 +155,8 @@ def tune_transformer(num_samples=8,
         mode="max",
         perturbation_interval=1,
         hyperparam_mutations={
-            "weight_decay": lambda: tune.uniform(0.0, 0.3).func(None),
-            "learning_rate": lambda: tune.uniform(1e-5, 5e-5).func(None),
+            "weight_decay": tune.uniform(0.0, 0.3),
+            "learning_rate": tune.uniform(1e-5, 5e-5),
             "per_gpu_train_batch_size": [16, 32, 64],
         })
 
diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index e0d670777e44..cf1c2b4725d3 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -1,6 +1,7 @@
 import logging
 import random
 from copy import copy
+from inspect import signature
 from numbers import Number
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
@@ -10,6 +11,16 @@
 
 
 class Domain:
+    """Base class to specify a type and valid range to sample parameters from.
+
+    This base class is implemented by parameter spaces, like float ranges
+    (``Float``), integer ranges (``Integer``), or categorical variables
+    (``Categorical``). The ``Domain`` object contains information about
+    valid values (e.g. minimum and maximum values), and exposes methods that
+    allow specification of specific samplers (e.g. ``uniform()`` or
+    ``loguniform()``).
+
+    """
     sampler = None
     default_sampler_cls = None
 
@@ -245,13 +256,16 @@ def sample(self,
                    domain: "Function",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            items = []
-            for i in range(size):
-                this_spec = spec[i] if isinstance(spec, list) else spec
-                items.append(domain.func(this_spec))
-            if len(items) == 1:
-                return items[0]
-            return items
+            pass_spec = len(signature(domain.func).parameters) > 0
+            if pass_spec:
+                items = [
+                    domain.func(spec[i] if isinstance(spec, list) else spec)
+                    for i in range(size)
+                ]
+            else:
+                items = [domain.func() for i in range(size)]
+
+            return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _CallSampler
 
@@ -283,6 +297,14 @@ def sample(self,
         return list(quantized)
 
 
+# TODO (krfricke): Remove tune.function
+def function(func):
+    logger.warning(
+        "DeprecationWarning: wrapping {} with tune.function() is no "
+        "longer needed".format(func))
+    return func
+
+
 def sample_from(func: Callable[[Dict], Any]):
     """Specify that tune should sample configuration values from this function.
 
diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py
index c1c00d8acc9d..ec7e96c5bf14 100644
--- a/python/ray/tune/tests/test_trial_scheduler.py
+++ b/python/ray/tune/tests/test_trial_scheduler.py
@@ -976,7 +976,7 @@ def testTuneSamplePrimitives(self):
     def testTuneSampleFromError(self):
         with self.assertRaises(ValueError):
             pbt, runner = self.basicSetup(hyperparam_mutations={
-                "float_factor": tune.sample_from(lambda _: 100.0)
+                "float_factor": tune.sample_from(lambda: 100.0)
             })
 
     def testPerturbationValues(self):
diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py
index 329f29a1e3c3..058da983d666 100644
--- a/python/ray/tune/tune.py
+++ b/python/ray/tune/tune.py
@@ -67,8 +67,6 @@ def _report_progress(runner, reporter, done=False):
 
 def run(run_or_experiment,
         name=None,
-        metric=None,
-        mode=None,
         stop=None,
         config=None,
         resources_per_trial=None,
@@ -146,8 +144,6 @@ def run(run_or_experiment,
             ``tune.register_trainable("lambda_id", lambda x: ...)``. You can
             then use ``tune.run("lambda_id")``.
         name (str): Name of experiment.
-        metric (str): Metric to optimize.
-        mode (str): One one ["min", "max"]. Direction to optimize.
         stop (dict | callable | :class:`Stopper`): Stopping criteria. If dict,
             the keys may be any field in the return result of 'train()',
             whichever is reached first. If function, it must take (trial_id,
@@ -332,10 +328,8 @@ def run(run_or_experiment,
     if not search_alg:
         search_alg = BasicVariantGenerator()
 
-    if mode:
-        assert mode in ["min", "max"], "`mode` must be one of [min, max]"
-
-    if config and not search_alg.set_search_properties(metric, mode, config):
+    # TODO (krfricke): Introduce metric/mode as top level API
+    if config and not search_alg.set_search_properties(None, None, config):
         logger.warning(
             "You passed a `config` parameter to `tune.run()`, but the "
             "search algorithm was already instantiated with a search space. "
@@ -421,8 +415,8 @@ def run(run_or_experiment,
     return ExperimentAnalysis(
         runner.checkpoint_file,
         trials=trials,
-        default_metric=metric,
-        default_mode=mode)
+        default_metric=None,
+        default_mode=None)
 
 
 def run_experiments(experiments,
diff --git a/python/ray/util/sgd/tf/examples/tensorflow_train_example.py b/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
index 8ca24f4998f3..41f4505ea864 100644
--- a/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
+++ b/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
@@ -95,8 +95,8 @@ def train_example(num_replicas=1, batch_size=128, use_gpu=False):
 
 def tune_example(num_replicas=1, use_gpu=False):
     config = {
-        "model_creator": tune.sample_from(simple_model),
-        "data_creator": tune.sample_from(simple_dataset),
+        "model_creator": simple_model,
+        "data_creator": simple_dataset,
         "num_replicas": num_replicas,
         "use_gpu": use_gpu,
         "trainer_config": create_config(batch_size=128)
diff --git a/python/setup.py b/python/setup.py
index 7aeeaab40b7b..bbe2a0be44b3 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -121,6 +121,7 @@
     "lz4",
     "opencv-python-headless<=4.3.0.36",
     "pyyaml",
+    "scipy",
 ]
 
 extras["streaming"] = []

From 1aaff6187c4e7b58a141cee4d2a95f56e6a997d5 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 2 Sep 2020 10:37:54 +0100
Subject: [PATCH 36/48] Fix sampling tests

---
 python/ray/tune/tests/test_sample.py | 36 +++++++---------------------
 1 file changed, 8 insertions(+), 28 deletions(-)

diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 81ed5703bb58..2ef926eb32b3 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -193,14 +193,9 @@ def testConvertAx(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
-        searcher = AxSearch()
+        searcher = AxSearch(metric="a", mode="max")
         analysis = tune.run(
-            _mock_objective,
-            metric="a",
-            mode="max",
-            config=config,
-            search_alg=searcher,
-            num_samples=1)
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
         trial = analysis.trials[0]
         assert trial.config["a"] in [2, 3, 4]
 
@@ -241,14 +236,9 @@ def testConvertBayesOpt(self):
         with self.assertRaises(ValueError):
             searcher.set_search_properties("none", "max", invalid_config)
 
-        searcher = BayesOptSearch()
+        searcher = BayesOptSearch(metric="a", mode="max")
         analysis = tune.run(
-            _mock_objective,
-            metric="a",
-            mode="max",
-            config=config,
-            search_alg=searcher,
-            num_samples=1)
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
         trial = analysis.trials[0]
         self.assertLess(trial.config["b"]["z"], 1e-2)
 
@@ -289,14 +279,9 @@ def testConvertHyperOpt(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
-        searcher = HyperOptSearch()
+        searcher = HyperOptSearch(metric="a", mode="max")
         analysis = tune.run(
-            _mock_objective,
-            metric="a",
-            mode="max",
-            config=config,
-            search_alg=searcher,
-            num_samples=1)
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
         trial = analysis.trials[0]
         assert trial.config["a"] in [2, 3, 4]
 
@@ -337,14 +322,9 @@ def testConvertOptuna(self):
         self.assertLess(1e-4, config1["b"]["z"])
         self.assertLess(config1["b"]["z"], 1e-2)
 
-        searcher = OptunaSearch()
+        searcher = OptunaSearch(metric="a", mode="max")
         analysis = tune.run(
-            _mock_objective,
-            metric="a",
-            mode="max",
-            config=config,
-            search_alg=searcher,
-            num_samples=1)
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
         trial = analysis.trials[0]
         assert trial.config["a"] in [2, 3, 4]
 

From 49c2b4fbe1a3b1a0a51a72d896b19831b373bfc0 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 2 Sep 2020 10:52:45 +0100
Subject: [PATCH 37/48] Fix experiment analysis tests

---
 .../tune/tests/test_experiment_analysis.py    | 42 ++++++++-----------
 1 file changed, 18 insertions(+), 24 deletions(-)

diff --git a/python/ray/tune/tests/test_experiment_analysis.py b/python/ray/tune/tests/test_experiment_analysis.py
index a2833318b46b..2aef0db1b0d1 100644
--- a/python/ray/tune/tests/test_experiment_analysis.py
+++ b/python/ray/tune/tests/test_experiment_analysis.py
@@ -9,7 +9,6 @@
 import ray
 from ray.tune import run, sample_from
 from ray.tune.examples.async_hyperband_example import MyTrainableClass
-from ray.tune.result import TRAINING_ITERATION
 
 
 class ExperimentAnalysisSuite(unittest.TestCase):
@@ -30,8 +29,6 @@ def run_test_exp(self):
         self.ea = run(
             MyTrainableClass,
             name=self.test_name,
-            metric=self.metric,
-            mode="max",
             local_dir=self.test_dir,
             stop={"training_iteration": 1},
             checkpoint_freq=1,
@@ -46,8 +43,6 @@ def nan_test_exp(self):
         nan_ea = run(
             lambda x: nan,
             name="testing_nan",
-            metric=self.metric,
-            mode="max",
             local_dir=self.test_dir,
             stop={"training_iteration": 1},
             checkpoint_freq=1,
@@ -78,66 +73,65 @@ def testTrialDataframe(self):
         self.assertEqual(trial_df.shape[0], 1)
 
     def testBestConfig(self):
-        best_config = self.ea.get_best_config()
+        best_config = self.ea.get_best_config(self.metric, mode="max")
         self.assertTrue(isinstance(best_config, dict))
         self.assertTrue("width" in best_config)
         self.assertTrue("height" in best_config)
 
     def testBestConfigNan(self):
         nan_ea = self.nan_test_exp()
-        best_config = nan_ea.get_best_config()
+        best_config = nan_ea.get_best_config(self.metric, mode="max")
         self.assertIsNone(best_config)
 
     def testBestLogdir(self):
-        logdir = self.ea.get_best_logdir()
+        logdir = self.ea.get_best_logdir(self.metric, mode="max")
         self.assertTrue(logdir.startswith(self.test_path))
-        logdir2 = self.ea.get_best_logdir(mode="min")
+        logdir2 = self.ea.get_best_logdir(self.metric, mode="min")
         self.assertTrue(logdir2.startswith(self.test_path))
         self.assertNotEquals(logdir, logdir2)
 
     def testBestLogdirNan(self):
         nan_ea = self.nan_test_exp()
-        logdir = nan_ea.get_best_logdir()
+        logdir = nan_ea.get_best_logdir(self.metric, mode="max")
         self.assertIsNone(logdir)
 
     def testGetTrialCheckpointsPathsByTrial(self):
-        best_trial = self.ea.get_best_trial()
-        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(
-            best_trial, TRAINING_ITERATION)
-        logdir = self.ea.get_best_logdir()
+        best_trial = self.ea.get_best_trial(self.metric, mode="max")
+        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(best_trial)
+        logdir = self.ea.get_best_logdir(self.metric, mode="max")
         expected_path = os.path.join(logdir, "checkpoint_1", "checkpoint")
         assert checkpoints_metrics[0][0] == expected_path
         assert checkpoints_metrics[0][1] == 1
 
     def testGetTrialCheckpointsPathsByPath(self):
-        logdir = self.ea.get_best_logdir()
-        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(
-            logdir, TRAINING_ITERATION)
+        logdir = self.ea.get_best_logdir(self.metric, mode="max")
+        checkpoints_metrics = self.ea.get_trial_checkpoints_paths(logdir)
         expected_path = os.path.join(logdir, "checkpoint_1/", "checkpoint")
         assert checkpoints_metrics[0][0] == expected_path
         assert checkpoints_metrics[0][1] == 1
 
     def testGetTrialCheckpointsPathsWithMetricByTrial(self):
-        best_trial = self.ea.get_best_trial()
-        paths = self.ea.get_trial_checkpoints_paths(best_trial)
-        logdir = self.ea.get_best_logdir()
+        best_trial = self.ea.get_best_trial(self.metric, mode="max")
+        paths = self.ea.get_trial_checkpoints_paths(best_trial, self.metric)
+        logdir = self.ea.get_best_logdir(self.metric, mode="max")
         expected_path = os.path.join(logdir, "checkpoint_1", "checkpoint")
         assert paths[0][0] == expected_path
         assert paths[0][1] == best_trial.metric_analysis[self.metric]["last"]
 
     def testGetTrialCheckpointsPathsWithMetricByPath(self):
-        best_trial = self.ea.get_best_trial()
-        logdir = self.ea.get_best_logdir()
+        best_trial = self.ea.get_best_trial(self.metric, mode="max")
+        logdir = self.ea.get_best_logdir(self.metric, mode="max")
         paths = self.ea.get_trial_checkpoints_paths(best_trial, self.metric)
         expected_path = os.path.join(logdir, "checkpoint_1", "checkpoint")
         assert paths[0][0] == expected_path
         assert paths[0][1] == best_trial.metric_analysis[self.metric]["last"]
 
     def testGetBestCheckpoint(self):
-        best_trial = self.ea.get_best_trial()
+        best_trial = self.ea.get_best_trial(self.metric, mode="max")
         checkpoints_metrics = self.ea.get_trial_checkpoints_paths(best_trial)
         expected_path = max(checkpoints_metrics, key=lambda x: x[1])[0]
-        best_checkpoint = self.ea.get_best_checkpoint(best_trial, self.metric)
+        best_checkpoint = self.ea.get_best_checkpoint(
+            best_trial, self.metric, mode="max")
         assert expected_path == best_checkpoint
 
     def testAllDataframes(self):

From 2b209cb51560accdea7537f07151ede3400c5e1c Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 2 Sep 2020 14:49:37 +0100
Subject: [PATCH 38/48] Fix tests and linting error

---
 doc/source/tune/api_docs/grid_random.rst | 2 +-
 python/ray/tune/__init__.py              | 9 +++++----
 python/ray/tune/suggest/ax.py            | 4 ++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/doc/source/tune/api_docs/grid_random.rst b/doc/source/tune/api_docs/grid_random.rst
index 32347eba23c4..7269507cccf4 100644
--- a/doc/source/tune/api_docs/grid_random.rst
+++ b/doc/source/tune/api_docs/grid_random.rst
@@ -207,7 +207,7 @@ tune.choice
 tune.sample_from
 ~~~~~~~~~~~~~~~~
 
-.. autoclass:: ray.tune.sample_from
+.. autofunction:: ray.tune.sample_from
 
 Grid Search API
 ---------------
diff --git a/python/ray/tune/__init__.py b/python/ray/tune/__init__.py
index e41544e7063e..b50d96c366cb 100644
--- a/python/ray/tune/__init__.py
+++ b/python/ray/tune/__init__.py
@@ -12,14 +12,15 @@
                               save_checkpoint, checkpoint_dir)
 from ray.tune.progress_reporter import (ProgressReporter, CLIReporter,
                                         JupyterNotebookReporter)
-from ray.tune.sample import (sample_from, uniform, quniform, choice, randint,
-                             qrandint, randn, qrandn, loguniform, qloguniform)
+from ray.tune.sample import (function, sample_from, uniform, quniform, choice,
+                             randint, qrandint, randn, qrandn, loguniform,
+                             qloguniform)
 
 __all__ = [
     "Trainable", "DurableTrainable", "TuneError", "grid_search",
     "register_env", "register_trainable", "run", "run_experiments", "Stopper",
-    "EarlyStopping", "Experiment", "sample_from", "track", "uniform",
-    "quniform", "choice", "randint", "qrandint", "randn", "qrandn",
+    "EarlyStopping", "Experiment", "function", "sample_from", "track",
+    "uniform", "quniform", "choice", "randint", "qrandint", "randn", "qrandn",
     "loguniform", "qloguniform", "ExperimentAnalysis", "Analysis",
     "CLIReporter", "JupyterNotebookReporter", "ProgressReporter", "report",
     "get_trial_dir", "get_trial_name", "get_trial_id", "make_checkpoint_dir",
diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index e51e5eb8c297..f33616b675e5 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -80,7 +80,7 @@ def easy_objective(config):
 
     def __init__(self,
                  space=None,
-                 metric="episode_reward_mean",
+                 metric=None,
                  mode="max",
                  parameter_constraints=None,
                  outcome_constraints=None,
@@ -107,7 +107,7 @@ def __init__(self,
         self._parameters = []
         self._live_trial_mapping = {}
 
-        if self._space:
+        if self._ax or self._space:
             self.setup_experiment()
 
     def setup_experiment(self):

From 1a78055c49e423c99fc8e0ef0909c1e7cf82234b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 2 Sep 2020 15:08:13 +0100
Subject: [PATCH 39/48] Removed unnecessary default_config attribute from
 OptunaSearch

---
 python/ray/tune/suggest/optuna.py | 38 ++++++++++++-------------------
 1 file changed, 15 insertions(+), 23 deletions(-)

diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index 7dab0152b6f2..eaa96085a02e 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -1,4 +1,3 @@
-import copy
 import logging
 import pickle
 from typing import Dict
@@ -6,8 +5,9 @@
 from ray.tune.result import TRAINING_ITERATION
 from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
     Quantized, Uniform
-from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars
+from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import unflatten_dict
 
 try:
     import optuna as ot
@@ -59,10 +59,6 @@ class OptunaSearch(Searcher):
             minimizing or maximizing the metric attribute.
         sampler (optuna.samplers.BaseSampler): Optuna sampler used to
             draw hyperparameter configurations. Defaults to ``TPESampler``.
-        base_config (dict): Base config dict that gets overwritten by the
-            Optuna sampling and is returned to each Tune trial. This could e.g.
-            contain static variables or configurations that should be passed
-            to each trial.
 
     Example:
 
@@ -84,14 +80,11 @@ class OptunaSearch(Searcher):
 
     """
 
-    def __init__(
-            self,
-            space=None,
-            metric="episode_reward_mean",
-            mode="max",
-            sampler=None,
-            base_config=None,
-    ):
+    def __init__(self,
+                 space=None,
+                 metric="episode_reward_mean",
+                 mode="max",
+                 sampler=None):
         assert ot is not None, (
             "Optuna must be installed! Run `pip install optuna`.")
         super(OptunaSearch, self).__init__(
@@ -102,8 +95,6 @@ def __init__(
 
         self._space = space
 
-        self._config = base_config or {}
-
         self._study_name = "optuna"  # Fixed study name for in-memory storage
         self._sampler = sampler or ot.samplers.TPESampler()
         assert isinstance(self._sampler, ot.samplers.BaseSampler), \
@@ -137,7 +128,6 @@ def set_search_properties(self, metric, mode, config):
         if mode:
             self._mode = mode
         self.setup_study(mode)
-        self._config = config
         return True
 
     def suggest(self, trial_id):
@@ -154,12 +144,14 @@ def suggest(self, trial_id):
             self._ot_trials[trial_id] = ot.trial.Trial(self._ot_study,
                                                        ot_trial_id)
         ot_trial = self._ot_trials[trial_id]
-        params = copy.copy(self._config)
-        for (fn, args, kwargs) in self._space:
-            param_name = args[0] if len(args) > 0 else kwargs["name"]
-            value = getattr(ot_trial, fn)(*args, **kwargs)  # Call Optuna trial
-            assign_value(params, param_name.split("/"), value)
-        return params
+
+        # getattr will fetch the trial.suggest_ function on Optuna trials
+        params = {
+            args[0] if len(args) > 0 else kwargs["name"]: getattr(
+                ot_trial, fn)(*args, **kwargs)
+            for (fn, args, kwargs) in self._space
+        }
+        return unflatten_dict(params)
 
     def on_trial_result(self, trial_id, result):
         metric = result[self.metric]

From cb87044c3e1bbde619d7122c3724dcf769a1fef8 Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 2 Sep 2020 17:34:25 +0100
Subject: [PATCH 40/48] Revert to set AxSearch default metric

---
 python/ray/tune/suggest/ax.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index f33616b675e5..9baac2b6d7ba 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -80,7 +80,7 @@ def easy_objective(config):
 
     def __init__(self,
                  space=None,
-                 metric=None,
+                 metric="episode_reward_mean",
                  mode="max",
                  parameter_constraints=None,
                  outcome_constraints=None,
@@ -135,14 +135,13 @@ def setup_experiment(self):
                 minimize=self._mode != "max")
         else:
             if any([
-                    self._space, self._metric, self._parameter_constraints,
+                    self._space, self._parameter_constraints,
                     self._outcome_constraints
             ]):
                 raise ValueError(
                     "If you create the Ax experiment yourself, do not pass "
                     "values for these parameters to `AxSearch`: {}.".format([
-                        "space", "metric", "parameter_constraints",
-                        "outcome_constraints"
+                        "space", "parameter_constraints", "outcome_constraints"
                     ]))
 
         exp = self._ax.experiment

From 3b043469e2fe26e152a6425caa2a4b529dc7c3a9 Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Wed, 2 Sep 2020 12:09:22 -0700
Subject: [PATCH 41/48] fix-min-max

---
 python/ray/tune/sample.py           | 108 ++++++++++++++--------------
 python/ray/tune/suggest/ax.py       |   8 +--
 python/ray/tune/suggest/bayesopt.py |   2 +-
 python/ray/tune/suggest/hyperopt.py |  30 ++++----
 python/ray/tune/suggest/optuna.py   |  13 ++--
 5 files changed, 81 insertions(+), 80 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index cf1c2b4725d3..b44bdf96176c 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -108,11 +108,11 @@ def sample(self,
                    domain: "Float",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            assert domain.min > float("-inf"), \
-                "Uniform needs a minimum bound"
-            assert domain.max < float("inf"), \
-                "Uniform needs a maximum bound"
-            items = np.random.uniform(domain.min, domain.max, size=size)
+            assert domain.lower > float("-inf"), \
+                "Uniform needs a lower bound"
+            assert domain.upper < float("inf"), \
+                "Uniform needs a upper bound"
+            items = np.random.uniform(domain.lower, domain.upper, size=size)
             return items if len(items) > 1 else domain.cast(items[0])
 
     class _LogUniform(LogUniform):
@@ -120,12 +120,12 @@ def sample(self,
                    domain: "Float",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            assert domain.min > 0, \
-                "LogUniform needs a minimum bound greater than 0"
-            assert 0 < domain.max < float("inf"), \
-                "LogUniform needs a maximum bound greater than 0"
-            logmin = np.log(domain.min) / np.log(self.base)
-            logmax = np.log(domain.max) / np.log(self.base)
+            assert domain.lower > 0, \
+                "LogUniform needs a lower bound greater than 0"
+            assert 0 < domain.upper < float("inf"), \
+                "LogUniform needs a upper bound greater than 0"
+            logmin = np.log(domain.lower) / np.log(self.base)
+            logmax = np.log(domain.upper) / np.log(self.base)
 
             items = self.base**(np.random.uniform(logmin, logmax, size=size))
             return items if len(items) > 1 else domain.cast(items[0])
@@ -135,45 +135,45 @@ def sample(self,
                    domain: "Float",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            assert not domain.min or domain.min == float("-inf"), \
+            assert not domain.lower or domain.lower == float("-inf"), \
                 "Normal sampling does not allow a lower value bound."
-            assert not domain.max or domain.max == float("inf"), \
+            assert not domain.upper or domain.upper == float("inf"), \
                 "Normal sampling does not allow a upper value bound."
             items = np.random.normal(self.mean, self.sd, size=size)
             return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _Uniform
 
-    def __init__(self, min: Optional[float], max: Optional[float]):
+    def __init__(self, lower: Optional[float], upper: Optional[float]):
         # Need to explicitly check for None
-        self.min = min if min is not None else float("-inf")
-        self.max = max if max is not None else float("inf")
+        self.lower = lower if lower is not None else float("-inf")
+        self.upper = upper if upper is not None else float("inf")
 
     def cast(self, value):
         return float(value)
 
     def uniform(self):
-        if not self.min > float("-inf"):
+        if not self.lower > float("-inf"):
             raise ValueError(
-                "Uniform requires a minimum bound. Make sure to set the "
-                "`min` parameter of `Float()`.")
-        if not self.max < float("inf"):
+                "Uniform requires a lower bound. Make sure to set the "
+                "`lower` parameter of `Float()`.")
+        if not self.upper < float("inf"):
             raise ValueError(
-                "Uniform requires a maximum bound. Make sure to set the "
-                "`max` parameter of `Float()`.")
+                "Uniform requires a upper bound. Make sure to set the "
+                "`upper` parameter of `Float()`.")
         new = copy(self)
         new.set_sampler(self._Uniform())
         return new
 
     def loguniform(self, base: float = 10):
-        if not self.min > 0:
+        if not self.lower > 0:
             raise ValueError(
-                "LogUniform requires a minimum bound greater than 0. "
-                "Make sure to set the `min` parameter of `Float()` correctly.")
-        if not 0 < self.max < float("inf"):
+                "LogUniform requires a lower bound greater than 0. "
+                f"Got: {self.lower}.")
+        if not 0 < self.upper < float("inf"):
             raise ValueError(
-                "LogUniform requires a minimum bound greater than 0. "
-                "Make sure to set the `max` parameter of `Float()` correctly.")
+                "LogUniform requires a upper bound greater than 0. "
+                f"Got: {self.upper}.")
         new = copy(self)
         new.set_sampler(self._LogUniform(base))
         return new
@@ -195,14 +195,14 @@ def sample(self,
                    domain: "Integer",
                    spec: Optional[Union[List[Dict], Dict]] = None,
                    size: int = 1):
-            items = np.random.randint(domain.min, domain.max, size=size)
+            items = np.random.randint(domain.lower, domain.upper, size=size)
             return items if len(items) > 1 else domain.cast(items[0])
 
     default_sampler_cls = _Uniform
 
-    def __init__(self, min, max):
-        self.min = min
-        self.max = max
+    def __init__(self, lower, upper):
+        self.lower = lower
+        self.upper = upper
 
     def cast(self, value):
         return int(value)
@@ -314,18 +314,18 @@ def sample_from(func: Callable[[Dict], Any]):
     return Function(func)
 
 
-def uniform(min: float, max: float):
-    """Sample a float value uniformly between ``min`` and ``max``.
+def uniform(lower: float, upper: float):
+    """Sample a float value uniformly between ``lower`` and ``upper``.
 
     Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
     ``np.random.uniform(1, 10))``
 
     """
-    return Float(min, max).uniform()
+    return Float(lower, upper).uniform()
 
 
-def quniform(min: float, max: float, q: float):
-    """Sample a quantized float value uniformly between ``min`` and ``max``.
+def quniform(lower: float, upper: float, q: float):
+    """Sample a quantized float value uniformly between ``lower`` and ``upper``.
 
     Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
     ``np.random.uniform(1, 10))``
@@ -334,22 +334,22 @@ def quniform(min: float, max: float, q: float):
     Quantization makes the upper bound inclusive.
 
     """
-    return Float(min, max).uniform().quantized(q)
+    return Float(lower, upper).uniform().quantized(q)
 
 
-def loguniform(min: float, max: float, base: float = 10):
+def loguniform(lower: float, upper: float, base: float = 10):
     """Sugar for sampling in different orders of magnitude.
 
     Args:
-        min (float): Lower boundary of the output interval (e.g. 1e-4)
-        max (float): Upper boundary of the output interval (e.g. 1e-2)
+        lower (float): Lower boundary of the output interval (e.g. 1e-4)
+        upper (float): Upper boundary of the output interval (e.g. 1e-2)
         base (int): Base of the log. Defaults to 10.
 
     """
-    return Float(min, max).loguniform(base)
+    return Float(lower, upper).loguniform(base)
 
 
-def qloguniform(min: float, max: float, q: float, base: float = 10):
+def qloguniform(lower: float, upper: float, q: float, base: float = 10):
     """Sugar for sampling in different orders of magnitude.
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
@@ -357,14 +357,14 @@ def qloguniform(min: float, max: float, q: float, base: float = 10):
     Quantization makes the upper bound inclusive.
 
     Args:
-        min (float): Lower boundary of the output interval (e.g. 1e-4)
-        max (float): Upper boundary of the output interval (e.g. 1e-2)
+        lower (float): Lower boundary of the output interval (e.g. 1e-4)
+        upper (float): Upper boundary of the output interval (e.g. 1e-2)
         q (float): Quantization number. The result will be rounded to an
             integer increment of this value.
         base (int): Base of the log. Defaults to 10.
 
     """
-    return Float(min, max).loguniform(base).quantized(q)
+    return Float(lower, upper).loguniform(base).quantized(q)
 
 
 def choice(categories: List):
@@ -377,22 +377,22 @@ def choice(categories: List):
     return Categorical(categories).uniform()
 
 
-def randint(min: int, max: int):
-    """Sample an integer value uniformly between ``min`` and ``max``.
+def randint(lower: int, upper: int):
+    """Sample an integer value uniformly between ``lower`` and ``upper``.
 
-    ``min`` is inclusive, ``max`` is exclusive.
+    ``lower`` is inclusive, ``upper`` is exclusive.
 
     Sampling from ``tune.randint(10)`` is equivalent to sampling from
     ``np.random.randint(10)``
 
     """
-    return Integer(min, max).uniform()
+    return Integer(lower, upper).uniform()
 
 
-def qrandint(min: int, max: int, q: int = 1):
-    """Sample an integer value uniformly between ``min`` and ``max``.
+def qrandint(lower: int, upper: int, q: int = 1):
+    """Sample an integer value uniformly between ``lower`` and ``upper``.
 
-    ``min`` is inclusive, ``max`` is also inclusive (!).
+    ``lower`` is inclusive, ``upper`` is also inclusive (!).
 
     The value will be quantized, i.e. rounded to an integer increment of ``q``.
     Quantization makes the upper bound inclusive.
@@ -401,7 +401,7 @@ def qrandint(min: int, max: int, q: int = 1):
     ``np.random.randint(10)``
 
     """
-    return Integer(min, max).uniform().quantized(q)
+    return Integer(lower, upper).uniform().quantized(q)
 
 
 def randn(mean: float = 0., sd: float = 1.):
diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 9baac2b6d7ba..35283e5b4ab3 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -223,7 +223,7 @@ def resolve_value(par, domain):
                     return {
                         "name": par,
                         "type": "range",
-                        "bounds": [domain.min, domain.max],
+                        "bounds": [domain.lower, domain.upper],
                         "value_type": "float",
                         "log_scale": True
                     }
@@ -231,7 +231,7 @@ def resolve_value(par, domain):
                     return {
                         "name": par,
                         "type": "range",
-                        "bounds": [domain.min, domain.max],
+                        "bounds": [domain.lower, domain.upper],
                         "value_type": "float",
                         "log_scale": False
                     }
@@ -240,7 +240,7 @@ def resolve_value(par, domain):
                     return {
                         "name": par,
                         "type": "range",
-                        "bounds": [domain.min, domain.max],
+                        "bounds": [domain.lower, domain.upper],
                         "value_type": "int",
                         "log_scale": True
                     }
@@ -248,7 +248,7 @@ def resolve_value(par, domain):
                     return {
                         "name": par,
                         "type": "range",
-                        "bounds": [domain.min, domain.max],
+                        "bounds": [domain.lower, domain.upper],
                         "value_type": "int",
                         "log_scale": False
                     }
diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index c88a701464e0..75dfa9bf30ef 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -355,7 +355,7 @@ def resolve_value(domain):
                     logger.warning(
                         "BayesOpt does not support specific sampling methods. "
                         "The {} sampler will be dropped.".format(sampler))
-                    return (domain.min, domain.max)
+                    return (domain.lower, domain.upper)
 
             raise ValueError("BayesOpt does not support parameters of type "
                              "`{}`".format(type(domain).__name__))
diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 9b28add05e71..940d09a52c14 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -295,15 +295,15 @@ def resolve_value(par, domain):
             if isinstance(domain, Float):
                 if isinstance(sampler, LogUniform):
                     if quantize:
-                        return hpo.hp.qloguniform(par, domain.min, domain.max,
-                                                  quantize)
-                    return hpo.hp.loguniform(par, np.log(domain.min),
-                                             np.log(domain.max))
+                        return hpo.hp.qloguniform(par, domain.lower,
+                                                  domain.upper, quantize)
+                    return hpo.hp.loguniform(par, np.log(domain.lower),
+                                             np.log(domain.upper))
                 elif isinstance(sampler, Uniform):
                     if quantize:
-                        return hpo.hp.quniform(par, domain.min, domain.max,
+                        return hpo.hp.quniform(par, domain.lower, domain.upper,
                                                quantize)
-                    return hpo.hp.uniform(par, domain.min, domain.max)
+                    return hpo.hp.uniform(par, domain.lower, domain.upper)
                 elif isinstance(sampler, Normal):
                     if quantize:
                         return hpo.hp.qnormal(par, sampler.mean, sampler.sd,
@@ -313,21 +313,21 @@ def resolve_value(par, domain):
             elif isinstance(domain, Integer):
                 if isinstance(sampler, Uniform):
                     if quantize:
-                        logger.warning(
+                        raise ValueError(
                             "HyperOpt does not support quantization for "
-                            "integer values. Dropped quantization.")
-                    if domain.min != 0:
-                        logger.warning(
+                            "integer values. Please use 'randint' instead.")
+                    if domain.lower != 0:
+                        raise ValueError(
                             "HyperOpt only allows integer sampling with "
-                            "lower bound 0. Dropped the lower bound {}".format(
-                                domain.min))
-                    if domain.max < 1:
+                            "lower bound 0. Please set the lower bound "
+                            f"to {domain.lower}")
+                    if domain.upper < 1:
                         raise ValueError(
                             "HyperOpt does not support integer sampling "
                             "of values lower than 0. Set your maximum range "
                             "to something above 0 (currently {})".format(
-                                domain.max))
-                    return hpo.hp.randint(par, domain.max)
+                                domain.upper))
+                    return hpo.hp.randint(par, domain.upper)
             elif isinstance(domain, Categorical):
                 if isinstance(sampler, Uniform):
                     return hpo.hp.choice(par, domain.categories)
diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py
index eaa96085a02e..2d184f78d0a7 100644
--- a/python/ray/tune/suggest/optuna.py
+++ b/python/ray/tune/suggest/optuna.py
@@ -206,13 +206,14 @@ def resolve_value(par, domain):
                         logger.warning(
                             "Optuna does not support both quantization and "
                             "sampling from LogUniform. Dropped quantization.")
-                    return param.suggest_loguniform(par, domain.min,
-                                                    domain.max)
+                    return param.suggest_loguniform(par, domain.lower,
+                                                    domain.upper)
                 elif isinstance(sampler, Uniform):
                     if quantize:
                         return param.suggest_discrete_uniform(
-                            par, domain.min, domain.max, quantize)
-                    return param.suggest_uniform(par, domain.min, domain.max)
+                            par, domain.lower, domain.upper, quantize)
+                    return param.suggest_uniform(par, domain.lower,
+                                                 domain.upper)
             elif isinstance(domain, Integer):
                 if isinstance(sampler, LogUniform):
                     if quantize:
@@ -220,10 +221,10 @@ def resolve_value(par, domain):
                             "Optuna does not support both quantization and "
                             "sampling from LogUniform. Dropped quantization.")
                     return param.suggest_int(
-                        par, domain.min, domain.max, log=True)
+                        par, domain.lower, domain.upper, log=True)
                 elif isinstance(sampler, Uniform):
                     return param.suggest_int(
-                        par, domain.min, domain.max, step=quantize or 1)
+                        par, domain.lower, domain.upper, step=quantize or 1)
             elif isinstance(domain, Categorical):
                 if isinstance(sampler, Uniform):
                     return param.suggest_categorical(par, domain.categories)

From b7f7f6742f0793ef44cf4db4ad42c00c58e0b54c Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Wed, 2 Sep 2020 12:52:01 -0700
Subject: [PATCH 42/48] fix

---
 python/ray/tune/suggest/hyperopt.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py
index 940d09a52c14..6cb6d1dfe517 100644
--- a/python/ray/tune/suggest/hyperopt.py
+++ b/python/ray/tune/suggest/hyperopt.py
@@ -313,14 +313,13 @@ def resolve_value(par, domain):
             elif isinstance(domain, Integer):
                 if isinstance(sampler, Uniform):
                     if quantize:
-                        raise ValueError(
+                        logger.warning(
                             "HyperOpt does not support quantization for "
-                            "integer values. Please use 'randint' instead.")
+                            "integer values. Reverting back to 'randint'.")
                     if domain.lower != 0:
                         raise ValueError(
                             "HyperOpt only allows integer sampling with "
-                            "lower bound 0. Please set the lower bound "
-                            f"to {domain.lower}")
+                            f"lower bound 0. Got: {domain.lower}.")
                     if domain.upper < 1:
                         raise ValueError(
                             "HyperOpt does not support integer sampling "

From 49bf48d25c0dda3d1b0e950c195e79b52e857782 Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Wed, 2 Sep 2020 12:55:56 -0700
Subject: [PATCH 43/48] nits

---
 python/ray/tune/examples/logging_example.py | 6 +++---
 python/ray/tune/suggest/ax.py               | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/python/ray/tune/examples/logging_example.py b/python/ray/tune/examples/logging_example.py
index d7449711369f..4f832c0dcf17 100755
--- a/python/ray/tune/examples/logging_example.py
+++ b/python/ray/tune/examples/logging_example.py
@@ -63,8 +63,8 @@ def load_checkpoint(self, checkpoint_path):
         trial_name_creator=trial_str_creator,
         loggers=[TestLogger],
         stop={"training_iteration": 1 if args.smoke_test else 99999},
+        search_alg=searcher,
         config={
-            "width": tune.sample_from(
-                lambda spec: 10 + int(90 * random.random())),
-            "height": tune.sample_from(lambda spec: int(100 * random.random()))
+            "width": tune.qrandint(0, 100),
+            "height": tune.loguniform(0.1, 100)
         })
diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py
index 35283e5b4ab3..e25556f86296 100644
--- a/python/ray/tune/suggest/ax.py
+++ b/python/ray/tune/suggest/ax.py
@@ -214,7 +214,7 @@ def convert_search_space(spec: Dict):
         def resolve_value(par, domain):
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):
-                logger.warning("Ax search does not support quantization. "
+                logger.warning("AxSearch does not support quantization. "
                                "Dropped quantization.")
                 sampler = sampler.sampler
 
@@ -260,7 +260,7 @@ def resolve_value(par, domain):
                         "values": domain.categories
                     }
 
-            raise ValueError("Ax search does not support parameters of type "
+            raise ValueError("AxSearch does not support parameters of type "
                              "`{}` with samplers of type `{}`".format(
                                  type(domain).__name__,
                                  type(domain.sampler).__name__))

From 84275d1c9cc4a29bca0b2c9f4fbe4a985083ed2b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Wed, 2 Sep 2020 21:01:26 +0100
Subject: [PATCH 44/48] Added function check, enhanced loguniform error message

---
 python/ray/tune/sample.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/python/ray/tune/sample.py b/python/ray/tune/sample.py
index b44bdf96176c..c0a85a62a5f5 100644
--- a/python/ray/tune/sample.py
+++ b/python/ray/tune/sample.py
@@ -168,12 +168,16 @@ def uniform(self):
     def loguniform(self, base: float = 10):
         if not self.lower > 0:
             raise ValueError(
-                "LogUniform requires a lower bound greater than 0. "
-                f"Got: {self.lower}.")
+                "LogUniform requires a lower bound greater than 0."
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead.")
         if not 0 < self.upper < float("inf"):
             raise ValueError(
                 "LogUniform requires a upper bound greater than 0. "
-                f"Got: {self.upper}.")
+                f"Got: {self.lower}. Did you pass a variable that has "
+                "been log-transformed? If so, pass the non-transformed value "
+                "instead.")
         new = copy(self)
         new.set_sampler(self._LogUniform(base))
         return new
@@ -270,6 +274,11 @@ def sample(self,
     default_sampler_cls = _CallSampler
 
     def __init__(self, func: Callable):
+        if len(signature(func).parameters) > 1:
+            raise ValueError(
+                "The function passed to a `Function` parameter must accept "
+                "either 0 or 1 parameters.")
+
         self.func = func
 
     def is_function(self):

From 0222f16ecef2c07e1ec5c2d4b4930cdee70ac637 Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Wed, 2 Sep 2020 13:02:10 -0700
Subject: [PATCH 45/48] fix-print

---
 python/ray/tune/suggest/bayesopt.py  | 1 -
 python/ray/tune/suggest/nevergrad.py | 1 -
 python/ray/tune/tests/test_sample.py | 1 -
 3 files changed, 3 deletions(-)

diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py
index 75dfa9bf30ef..7aa4e2bbbb5d 100644
--- a/python/ray/tune/suggest/bayesopt.py
+++ b/python/ray/tune/suggest/bayesopt.py
@@ -328,7 +328,6 @@ def restore(self, checkpoint_path):
 
     @staticmethod
     def convert_search_space(spec: Dict):
-        print(spec)
         spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
diff --git a/python/ray/tune/suggest/nevergrad.py b/python/ray/tune/suggest/nevergrad.py
index 17e4defaeba8..6208657225a8 100644
--- a/python/ray/tune/suggest/nevergrad.py
+++ b/python/ray/tune/suggest/nevergrad.py
@@ -115,7 +115,6 @@ def suggest(self, trial_id):
         # in v0.2.0+, output of ask() is a Candidate,
         # with fields args and kwargs
         if not suggested_config.kwargs:
-            print(suggested_config.args, suggested_config.kwargs)
             return dict(zip(self._parameters, suggested_config.args[0]))
         else:
             return suggested_config.kwargs
diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py
index 2ef926eb32b3..63080f06b234 100644
--- a/python/ray/tune/tests/test_sample.py
+++ b/python/ray/tune/tests/test_sample.py
@@ -215,7 +215,6 @@ def testConvertBayesOpt(self):
 
         config = {"b": {"z": tune.sample.Float(1e-4, 1e-2).loguniform()}}
         bayesopt_config = {"b/z": (1e-4, 1e-2)}
-
         converted_config = BayesOptSearch.convert_search_space(config)
 
         searcher1 = BayesOptSearch(space=converted_config, metric="none")

From d72ccf1252f21d064be775609abf0b8f40ab5f22 Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Wed, 2 Sep 2020 14:59:28 -0700
Subject: [PATCH 46/48] fix

---
 python/ray/tune/examples/logging_example.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/python/ray/tune/examples/logging_example.py b/python/ray/tune/examples/logging_example.py
index 4f832c0dcf17..b94e28867f0f 100755
--- a/python/ray/tune/examples/logging_example.py
+++ b/python/ray/tune/examples/logging_example.py
@@ -63,8 +63,7 @@ def load_checkpoint(self, checkpoint_path):
         trial_name_creator=trial_str_creator,
         loggers=[TestLogger],
         stop={"training_iteration": 1 if args.smoke_test else 99999},
-        search_alg=searcher,
         config={
-            "width": tune.qrandint(0, 100),
-            "height": tune.loguniform(0.1, 100)
+            "width": tune.randint(10, 100),
+            "height": tune.loguniform(10, 100)
         })

From 3070cdaf2cf174ca95d6af93347351a1a34c9676 Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Wed, 2 Sep 2020 17:49:49 -0700
Subject: [PATCH 47/48] fix

---
 python/ray/tune/examples/logging_example.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/ray/tune/examples/logging_example.py b/python/ray/tune/examples/logging_example.py
index b94e28867f0f..c0bf7eda4348 100755
--- a/python/ray/tune/examples/logging_example.py
+++ b/python/ray/tune/examples/logging_example.py
@@ -3,8 +3,6 @@
 import argparse
 import json
 import os
-import random
-
 import numpy as np
 
 from ray import tune

From b53b653f1e8077aa3cd0a3e5497d8dce74bd7f0b Mon Sep 17 00:00:00 2001
From: Kai Fricke <kai@anyscale.com>
Date: Thu, 3 Sep 2020 10:50:58 +0100
Subject: [PATCH 48/48] Raise if unresolved values are in config and search
 space is already set

---
 python/ray/tune/suggest/variant_generator.py |  4 ++++
 python/ray/tune/tune.py                      | 13 ++++++++-----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/python/ray/tune/suggest/variant_generator.py b/python/ray/tune/suggest/variant_generator.py
index 5437a944db8a..a539c7b2c481 100644
--- a/python/ray/tune/suggest/variant_generator.py
+++ b/python/ray/tune/suggest/variant_generator.py
@@ -282,6 +282,10 @@ def _unresolved_values(spec):
     return _split_resolved_unresolved_values(spec)[1]
 
 
+def has_unresolved_values(spec):
+    return True if _unresolved_values(spec) else False
+
+
 class _UnresolvedAccessGuard(dict):
     def __init__(self, *args, **kwds):
         super(_UnresolvedAccessGuard, self).__init__(*args, **kwds)
diff --git a/python/ray/tune/tune.py b/python/ray/tune/tune.py
index 058da983d666..fba669a1b3b6 100644
--- a/python/ray/tune/tune.py
+++ b/python/ray/tune/tune.py
@@ -5,6 +5,7 @@
 from ray.tune.analysis import ExperimentAnalysis
 from ray.tune.suggest import BasicVariantGenerator, SearchGenerator
 from ray.tune.suggest.suggestion import Searcher
+from ray.tune.suggest.variant_generator import has_unresolved_values
 from ray.tune.trial import Trial
 from ray.tune.trainable import Trainable
 from ray.tune.ray_trial_executor import RayTrialExecutor
@@ -330,11 +331,13 @@ def run(run_or_experiment,
 
     # TODO (krfricke): Introduce metric/mode as top level API
     if config and not search_alg.set_search_properties(None, None, config):
-        logger.warning(
-            "You passed a `config` parameter to `tune.run()`, but the "
-            "search algorithm was already instantiated with a search space. "
-            "Any search definitions in the `config` passed to `tune.run()` "
-            "will be ignored.")
+        if has_unresolved_values(config):
+            raise ValueError(
+                "You passed a `config` parameter to `tune.run()` with "
+                "unresolved parameters, but the search algorithm was already "
+                "instantiated with a search space. Make sure that `config` "
+                "does not contain any more parameter definitions - include "
+                "them in the search algorithm's search space if necessary.")
 
     runner = TrialRunner(
         search_alg=search_alg,