support string alg in tune (microsoft#1093)

* support string alg in tune

* add test, enforce string feasible, support lexico in set_search_priorities in CFO

* fix bug

* fix bug

* fix bug

* fix bug

* fix bugs

* fix yiran

---------

Co-authored-by: “skzhang1” <“[email protected]”>

Author: skzhang1

flaml/tune/tune.py

@@ -344,7 +344,7 @@ def easy_objective(config):
                 # do cleanup operation here
                 return
     ```
-        search_alg: An instance of BlendSearch as the search algorithm
+        search_alg: An instance/string of the search algorithm
             to be used. The same instance can be used for iterative tuning.
             e.g.,
 
@@ -481,12 +481,25 @@ def easy_objective(config):
         else:
             logger.setLevel(logging.CRITICAL)
 
-    from .searcher.blendsearch import BlendSearch, CFO
+    from .searcher.blendsearch import BlendSearch, CFO, RandomSearch
 
     if lexico_objectives is not None:
-        logger.warning("If lexico_objectives is not None, search_alg is forced to be CFO")
-        search_alg = None
-    if search_alg is None:
+        if "modes" not in lexico_objectives.keys():
+            lexico_objectives["modes"] = ["min"] * len(lexico_objectives["metrics"])
+        for t_metric, t_mode in zip(lexico_objectives["metrics"], lexico_objectives["modes"]):
+            if t_metric not in lexico_objectives["tolerances"].keys():
+                lexico_objectives["tolerances"][t_metric] = 0
+            if t_metric not in lexico_objectives["targets"].keys():
+                lexico_objectives["targets"][t_metric] = -float("inf") if t_mode == "min" else float("inf")
+    if search_alg is None or isinstance(search_alg, str):
+        if isinstance(search_alg, str):
+            assert search_alg in [
+                "BlendSearch",
+                "CFO",
+                "CFOCat",
+                "RandomSearch",
+            ], f"search_alg={search_alg} is not recognized. 'BlendSearch', 'CFO', 'CFOcat' and 'RandomSearch' are supported."
+
         flaml_scheduler_resource_attr = (
             flaml_scheduler_min_resource
         ) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None
@@ -500,20 +513,30 @@ def easy_objective(config):
             flaml_scheduler_max_resource = max_resource
             flaml_scheduler_reduction_factor = reduction_factor
             scheduler = None
-        if lexico_objectives is None:
-            try:
-                import optuna as _
+        if lexico_objectives:
+            # TODO: Modify after supporting BlendSearch in lexicographic optimization
+            SearchAlgorithm = CFO
+            logger.info(
+                f"Using search algorithm {SearchAlgorithm.__name__} for lexicographic optimization. Note that when providing other search algorithms, we use CFO instead temporarily."
+            )
+            metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
+        else:
+            if not search_alg or search_alg == "BlendSearch":
+                try:
+                    import optuna as _
 
-                SearchAlgorithm = BlendSearch
+                    SearchAlgorithm = BlendSearch
+                    logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
+                except ImportError:
+                    if search_alg == "BlendSearch":
+                        raise ValueError("To use BlendSearch, run: pip install flaml[blendsearch]")
+                    else:
+                        SearchAlgorithm = CFO
+                        logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
+            else:
+                SearchAlgorithm = locals()[search_alg]
                 logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
-            except ImportError:
-                SearchAlgorithm = CFO
-                logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
             metric = metric or DEFAULT_METRIC
-        else:
-            SearchAlgorithm = CFO
-            logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
-            metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
         search_alg = SearchAlgorithm(
             metric=metric,
             mode=mode,
@@ -535,8 +558,12 @@ def easy_objective(config):
         )
     else:
         if metric is None or mode is None:
-            metric = metric or search_alg.metric or DEFAULT_METRIC
-            mode = mode or search_alg.mode
+            if lexico_objectives:
+                metric = lexico_objectives["metrics"][0] or metric or search_alg.metric or DEFAULT_METRIC
+                mode = lexico_objectives["modes"][0] or mode or search_alg.mode
+            else:
+                metric = metric or search_alg.metric or DEFAULT_METRIC
+                mode = mode or search_alg.mode
         if ray_available and use_ray:
             if ray_version.startswith("1."):
                 from ray.tune.suggest import ConcurrencyLimiter
@@ -555,6 +582,13 @@ def easy_objective(config):
         ):
             search_alg.use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
         searcher = search_alg.searcher if isinstance(search_alg, ConcurrencyLimiter) else search_alg
+        if lexico_objectives:
+            # TODO: Modify after supporting BlendSearch in lexicographic optimization
+            assert search_alg.__class__.__name__ in [
+                "CFO",
+            ], "If lexico_objectives is not None, the search_alg must be CFO for now."
+            search_alg.lexico_objective = lexico_objectives
+
         if isinstance(searcher, BlendSearch):
             setting = {}
             if time_budget_s:

setup.py

@@ -85,7 +85,7 @@
             "tensorboardX==2.6",  # test_forecast_panel
             "requests<2.29.0",  # https://github.com/docker/docker-py/issues/3113
             "packaging",
-            "pydantic",
+            "pydantic==1.10.9",
             "sympy",
             "wolframalpha",
             "joblib<1.3.0",  # temp solution for joblib 1.3.0 issue, no need once https://github.com/joblib/joblib-spark/pull/48 is merged
@@ -137,7 +137,7 @@
         "benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3", "pandas==1.1.4"],
         "openai": ["openai==0.27.8", "diskcache"],
         "autogen": ["openai==0.27.8", "diskcache", "docker"],
-        "mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic", "wolframalpha"],
+        "mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic==1.10.9", "wolframalpha"],
         "synapse": [
             "joblibspark>=0.5.0",
             "optuna==2.8.0",

test/tune/test_tune.py

@@ -1,13 +1,14 @@
 """Require: pip install flaml[test,ray]
 """
-from flaml import BlendSearch
+from flaml import BlendSearch, CFO
 import time
 import os
 from sklearn.model_selection import train_test_split
 import sklearn.metrics
 import sklearn.datasets
 import xgboost as xgb
 import logging
+import math
 
 try:
     from ray.tune.integration.xgboost import TuneReportCheckpointCallback
@@ -20,6 +21,32 @@
 logger.setLevel(logging.INFO)
 
 
+def _BraninCurrin(config):
+    # Rescale brain
+    x_1 = 15 * config["x1"] - 5
+    x_2 = 15 * config["x2"]
+    # Brain function
+    t1 = x_2 - 5.1 / (4 * math.pi**2) * x_1**2 + 5 / math.pi * x_1 - 6
+    t2 = 10 * (1 - 1 / (8 * math.pi)) * math.cos(x_1)
+    brain_result = t1**2 + t2 + 10
+    # Currin function
+    xc_1 = config["x1"]
+    xc_2 = config["x2"]
+    factor1 = 1 - math.exp(-1 / (2 * xc_2))
+    numer = 2300 * pow(xc_1, 3) + 1900 * pow(xc_1, 2) + 2092 * xc_1 + 60
+    denom = 100 * pow(xc_1, 3) + 500 * pow(xc_1, 2) + 4 * xc_1 + 20
+    currin_result = factor1 * numer / denom
+    return {"brain": brain_result, "currin": currin_result}
+
+
+def _easy_objective(config):
+    # Hyperparameters
+    width, height, step = config["width"], config["height"], config["steps"]
+
+    # get_result
+    return {"mean_loss": (0.1 + width * step / 100) ** (-1) + height * 0.1}
+
+
 def test_nested_run():
     from flaml import AutoML, tune
 
@@ -352,6 +379,72 @@ def evaluate_config_empty(config):
     )
 
 
+def test_passing_search_alg():
+    from flaml import tune
+
+    # search_space
+    so_search_space = {
+        "steps": 100,
+        "width": tune.uniform(0, 20),
+        "height": tune.uniform(-100, 100),
+    }
+    mo_search_space = {
+        "x1": tune.uniform(lower=0.000001, upper=1.0),
+        "x2": tune.uniform(lower=0.000001, upper=1.0),
+    }
+
+    # lexicographic objectives
+    lexico_objectives = {}
+    lexico_objectives["metrics"] = ["brain", "currin"]
+    lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
+    lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
+    lexico_objectives["modes"] = ["min", "min"]
+
+    ## Passing search_alg through string
+    # Non lexico tune
+    tune.run(
+        _easy_objective,
+        search_alg="BlendSearch",
+        metric="mean_loss",
+        mode="min",
+        num_samples=10,
+        config=so_search_space,
+    )
+    # lexico tune
+    tune.run(
+        _BraninCurrin, search_alg="CFO", num_samples=10, config=mo_search_space, lexico_objectives=lexico_objectives
+    )
+    tune.run(
+        _BraninCurrin,
+        search_alg="BlendSearch",
+        num_samples=10,
+        config=mo_search_space,
+        lexico_objectives=lexico_objectives,
+    )
+
+    ## Passing search_alg through instance
+    so_bs = BlendSearch(time_budget_s=5, metric="mean_loss", mode="min")
+    # TODO: We will change CFO into blendsearch in the future
+    mo_bs = CFO(time_budget_s=5)
+    # Non lexico tune
+    tune.run(
+        _easy_objective,
+        search_alg=so_bs,
+        metric="mean_loss",
+        mode="min",
+        num_samples=10,
+        config=so_search_space,
+    )
+    # lexico tune
+    tune.run(
+        _BraninCurrin,
+        search_alg=mo_bs,
+        num_samples=10,
+        config=mo_search_space,
+        lexico_objectives=lexico_objectives,
+    )
+
+
 def test_xgboost_bs():
     _test_xgboost()