diff --git a/.gitignore b/.gitignore index 69ba4ec92..410f99bc5 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ jahs_bench_data/ # Jupyter .ipynb_checkpoints/ +*.ipynb # MacOS *.DS_Store diff --git a/README.md b/README.md index 5356de78f..a30c0cb9e 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ Using `neps` always follows the same pattern: 1. Define a `evaluate_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations for your problem. -1. Define a search space named `pipeline_space` of those Parameters e.g. via a dictionary -1. Call `neps.run(evaluate_pipeline, pipeline_space)` +2. Define a `pipeline_space` of those Parameters +3. Call `neps.run(evaluate_pipeline, pipeline_space)` In code, the usage pattern can look like this: @@ -53,7 +53,7 @@ import logging logging.basicConfig(level=logging.INFO) # 1. Define a function that accepts hyperparameters and computes the validation error -def evaluate_pipeline(lr: float, alpha: int, optimizer: str) -> float: +def evaluate_pipeline(lr: float, alpha: int, optimizer: str): # Create your model model = MyModel(lr=lr, alpha=alpha, optimizer=optimizer) @@ -63,21 +63,20 @@ def evaluate_pipeline(lr: float, alpha: int, optimizer: str) -> float: # 2. Define a search space of parameters; use the same parameter names as in evaluate_pipeline -pipeline_space = dict( - lr=neps.Float( +class ExampleSpace(neps.PipelineSpace): + lr = neps.Float( lower=1e-5, upper=1e-1, log=True, # Log spaces - prior=1e-3, # Incorporate you knowledge to help optimization - ), - alpha=neps.Integer(lower=1, upper=42), - optimizer=neps.Categorical(choices=["sgd", "adam"]) -) + prior=1e-3, # Incorporate your knowledge to help optimization + ) + alpha = neps.Integer(lower=1, upper=42) + optimizer = neps.Categorical(choices=["sgd", "adam"]) # 3. Run the NePS optimization neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=ExampleSpace(), root_directory="path/to/save/results", # Replace with the actual path. evaluations_to_spend=100, ) @@ -93,6 +92,8 @@ Discover how NePS works through these examples: - **[Utilizing Expert Priors for Hyperparameters](neps_examples/efficiency/expert_priors_for_hyperparameters.py)**: Learn how to incorporate expert priors for more efficient hyperparameter selection. +- **[Benefiting NePS State and Optimizers with custom runtime](neps_examples/experimental/ask_and_tell_example.py)**: Learn how to use AskAndTell, an advanced tool for leveraging optimizers and states while enabling a custom runtime for trial execution. + - **[Additional NePS Examples](neps_examples/)**: Explore more examples, including various use cases and advanced configurations in NePS. ## Contributing diff --git a/docs/getting_started.md b/docs/getting_started.md index 6f69d5a95..5cbcdf9da 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -13,18 +13,17 @@ pip install neural-pipeline-search ## The 3 Main Components -1. **Establish a [`pipeline_space`](reference/pipeline_space.md)**: +1. **Establish a [`pipeline_space=`](reference/neps_spaces.md)**: ```python -pipeline_space={ - "some_parameter": (0.0, 1.0), # float - "another_parameter": (0, 10), # integer - "optimizer": ["sgd", "adam"], # categorical - "epoch": neps.Integer(lower=1, upper=100, is_fidelity=True), - "learning_rate": neps.Float(lower=1e-5, upper=1, log=True), - "alpha": neps.Float(lower=0.1, upper=1.0, prior=0.99, prior_confidence="high") -} - +class ExampleSpace(neps.PipelineSpace): + # Define the parameters of your search space + some_parameter = neps.Float(lower=0.0, upper=1.0) # float + another_parameter = neps.Integer(lower=0, upper=10) # integer + optimizer = neps.Categorical(choices=("sgd", "adam")) # categorical + epoch = neps.IntegerFidelity(lower=1, upper=100) + learning_rate = neps.Float(lower=1e-5, upper=1, log=True) + alpha = neps.Float(lower=0.1, upper=1.0, prior=0.99, prior_confidence="high") ``` 2. **Define an `evaluate_pipeline()` function**: @@ -42,7 +41,7 @@ def evaluate_pipeline(some_parameter: float, 3. **Execute with [`neps.run()`](reference/neps_run.md)**: ```python -neps.run(evaluate_pipeline, pipeline_space) +neps.run(evaluate_pipeline, ExampleSpace()) ``` --- @@ -52,14 +51,14 @@ neps.run(evaluate_pipeline, pipeline_space) The [reference](reference/neps_run.md) section provides detailed information on the individual components of NePS. 1. How to use the [**`neps.run()`** function](reference/neps_run.md) to start the optimization process. -2. The different [search space](reference/pipeline_space.md) options available. +2. The different [search space](reference/neps_spaces.md) options available. 3. How to choose and configure the [optimizer](reference/optimizers.md) used. 4. How to define the [`evaluate_pipeline()` function](reference/evaluate_pipeline.md). 5. How to [analyze](reference/analyse.md) the optimization runs. Or discover the features of NePS through these practical examples: -* **[Hyperparameter Optimization (HPO)](examples/basic_usage/hyperparameters.md)**: +* **[Hyperparameter Optimization (HPO)](examples/basic_usage/1_hyperparameters.md)**: Learn the essentials of hyperparameter optimization with NePS. * **[Multi-Fidelity Optimization](examples/efficiency/multi_fidelity.md)**: diff --git a/docs/index.md b/docs/index.md index e0341e664..dffc64211 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,33 +59,26 @@ import logging # 1. Define a function that accepts hyperparameters and computes the validation error -def evaluate_pipeline( - hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str -) -> dict: +def evaluate_pipeline(hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str): # Create your model model = MyModel(architecture_parameter) # Train and evaluate the model with your training pipeline - validation_error = train_and_eval( - model, hyperparameter_a, hyperparameter_b - ) + validation_error = train_and_eval(model, hyperparameter_a, hyperparameter_b) return validation_error # 2. Define a search space of parameters; use the same parameter names as in evaluate_pipeline -pipeline_space = dict( - hyperparameter_a=neps.Float( - lower=0.001, upper=0.1, log=True # The search space is sampled in log space - ), - hyperparameter_b=neps.Integer(lower=1, upper=42), - architecture_parameter=neps.Categorical(["option_a", "option_b"]), -) +class ExampleSpace(neps.PipelineSpace): + hyperparameter_a = neps.Float(lower=0.001, upper=0.1, log=True) # Log scale parameter + hyperparameter_b = neps.Integer(lower=1, upper=42) + architecture_parameter = neps.Categorical(choices=("option_a", "option_b")) # 3. Run the NePS optimization logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=ExampleSpace(), root_directory="path/to/save/results", # Replace with the actual path. evaluations_to_spend=100, ) @@ -98,7 +91,7 @@ neps.run( Discover how NePS works through these examples: -- **[Hyperparameter Optimization](examples/basic_usage/hyperparameters.md)**: Learn the essentials of hyperparameter optimization with NePS. +- **[Hyperparameter Optimization](examples/basic_usage/1_hyperparameters.md)**: Learn the essentials of hyperparameter optimization with NePS. - **[Multi-Fidelity Optimization](examples/efficiency/multi_fidelity.md)**: Understand how to leverage multi-fidelity optimization for efficient model tuning. diff --git a/docs/reference/evaluate_pipeline.md b/docs/reference/evaluate_pipeline.md index 034f16748..610edefc1 100644 --- a/docs/reference/evaluate_pipeline.md +++ b/docs/reference/evaluate_pipeline.md @@ -1,4 +1,4 @@ -# The `evaluate_pipeline` function +# The `evaluate_pipeline` function > **TL;DR** > *Sync*: return a scalar or a dict ⟶ NePS records it automatically. @@ -6,33 +6,33 @@ --- -## 1  Return types +## 1 Return types | Allowed return | When to use | Minimal example | | -------------- | ------------------------------------------- | -------------------------------------------- | | **Scalar** | simple objective, single fidelity | `return loss` | | **Dict** | need cost/extra metrics | `{"objective_to_minimize": loss, "cost": 3}` | -| **`None`** | you launch the job elsewhere (SLURM, k8s …) | *see § 3 Async* | +| **`None`** | you launch the job elsewhere (SLURM, k8s …) | *see § 3 Async* | All other values raise a `TypeError` inside NePS. -## 2  Result dictionary keys +## 2 Result dictionary keys | key | purpose | required? | | ----------------------- | ---------------------------------------------------------------------------- | ----------------------------- | | `objective_to_minimize` | scalar NePS will minimise | **yes** | -| `cost` | wall‑clock, GPU‑hours, … — only if you passed `cost_to_spend` to `neps.run` | yes *iff* cost budget enabled | +| `cost` | wall‑clock, GPU‑hours, … — only if you passed `cost_to_spend` to `neps.run` | yes *iff* cost budget enabled | | `learning_curve` | list/np.array of intermediate objectives | optional | | `extra` | any JSON‑serialisable blob | optional | | `exception` | any Exception illustrating the error in evaluation | optional | -> **Tip**  Return exactly what you need; extra keys are preserved in the trial’s `report.yaml`. +> **Tip** Return exactly what you need; extra keys are preserved in the trial’s `report.yaml`. --- -## 3  Asynchronous evaluation (advanced) +## 3 Asynchronous evaluation (advanced) -### 3.1 Design +### 3.1 Design 1. **The Python side** (your `evaluate_pipeline` function) @@ -51,9 +51,9 @@ All other values raise a `TypeError` inside NePS. when it finishes. This writes `report.yaml` and marks the trial *SUCCESS* / *CRASHED*. -### 3.2 Code walk‑through +### 3.2 Code walk‑through -`submit.py` – called by NePS synchronously +`submit.py` – called by NePS synchronously ```python from pathlib import Path @@ -67,7 +67,7 @@ def evaluate_pipeline( learning_rate: float, optimizer: str, ): - # 1) write a Slurm script + # 1) write a Slurm script script = f"""#!/bin/bash #SBATCH --time=0-00:10 #SBATCH --job-name=trial_{pipeline_id} @@ -82,15 +82,15 @@ python run_pipeline.py \ --root_dir {root_directory} """) - # 2) submit and RETURN None (async) + # 2) submit and RETURN None (async) script_path = pipeline_directory / "submit.sh" script_path.write_text(script) os.system(f"sbatch {script_path}") - return None # ⟵ signals async mode + return None # ⟵ signals async mode ``` -`run_pipeline.py` – executed on the compute node +`run_pipeline.py` – executed on the compute node ```python import argparse, json, time, neps @@ -124,8 +124,6 @@ neps.save_pipeline_results( ) ``` -### 3.3 Why this matters - * No worker idles while your job is in the queue ➜ better throughput. * Crashes inside the job still mark the trial *CRASHED* instead of hanging. * Compatible with Successive‑Halving/ASHA — NePS just waits for `report.yaml`. @@ -134,7 +132,7 @@ neps.save_pipeline_results( * When using async approach, one worker, may create as many trials as possible, of course that in `Slurm` or other workload managers it's impossible to overload the system because of limitations set for each user, but if you want to control resources used for optimization, it's crucial to set `evaluations_to_spend` when calling `neps.run`. -## 4  Extra injected arguments +## 4 Extra injected arguments | name | provided when | description | | ----------------------------- | ----------------------- | ---------------------------------------------------------- | @@ -146,7 +144,7 @@ Use them to handle warm‑starts, logging and result persistence. --- -## 5  Checklist +## 5 Checklist * [x] Return scalar **or** dict **or** `None`. * [x] Include `cost` when using cost budgets. diff --git a/docs/reference/neps_run.md b/docs/reference/neps_run.md index 16fb76642..a7f419d34 100644 --- a/docs/reference/neps_run.md +++ b/docs/reference/neps_run.md @@ -17,15 +17,15 @@ import neps def evaluate_pipeline(learning_rate: float, epochs: int) -> float: # Your code here - return loss +class ExamplePipeline(neps.PipelineSpace): + learning_rate = neps.Float(1e-3, 1e-1, log=True) + epochs = neps.IntegerFidelity(10, 100) + neps.run( evaluate_pipeline=evaluate_pipeline, # (1)! - pipeline_space={, # (2)! - "learning_rate": neps.Float(1e-3, 1e-1, log=True), - "epochs": neps.Integer(10, 100) - }, + pipeline_space=ExamplePipeline(), # (2)! root_directory="path/to/result_dir" # (3)! ) ``` @@ -33,16 +33,16 @@ neps.run( 1. The objective function, targeted by NePS for minimization, by evaluation various configurations. It requires these configurations as input and should return either a dictionary or a sole loss value as the output. 2. This defines the search space for the configurations from which the optimizer samples. - It accepts either a dictionary with the configuration names as keys, a path to a YAML configuration file, or a [`configSpace.ConfigurationSpace`](https://automl.github.io/ConfigSpace/) object. - For comprehensive information and examples, please refer to the detailed guide available [here](../reference/pipeline_space.md) + It accepts a class instance inheriting from `neps.PipelineSpace` or a [`configSpace.ConfigurationSpace`](https://automl.github.io/ConfigSpace/) object. + For comprehensive information and examples, please refer to the detailed guide available [here](../reference/neps_spaces.md) 3. The directory path where the information about the optimization and its progress gets stored. This is also used to synchronize multiple calls to `neps.run()` for parallelization. See the following for more: -* What kind of [pipeline space](../reference/pipeline_space.md) can you define? -* What goes in and what goes out of [`evaluate_pipeline()`](../reference/evaluate_pipeline.md)? +* What kind of [pipeline space](../reference/neps_spaces.md) can you define? +* What goes in and what goes out of [`evaluate_pipeline()`](../reference/neps_run.md)? ## Budget, how long to run? To define a budget, provide `evaluations_to_spend=` to [`neps.run()`][neps.api.run], @@ -69,7 +69,7 @@ neps.run( 2. Prevents the initiation of new evaluations once this cost threshold is surpassed. This can be any kind of cost metric you like, such as time, energy, or monetary, as long as you can calculate it. This requires adding a cost value to the output of the `evaluate_pipeline` function, for example, return `#!python {'objective_to_minimize': loss, 'cost': cost}`. - For more details, please refer [here](../reference/evaluate_pipeline.md) + For more details, please refer [here](../reference/neps_spaces.md) ## Getting some feedback, logging NePS will not print anything to the console. To view the progress of workers, @@ -99,13 +99,55 @@ def run(learning_rate: float, epochs: int) -> float: return {"objective_to_minimize": loss, "cost": duration} neps.run( - # Increase the total number of trials from 10 as set previously to 50 + # Increase the total number of trials from 10 as set previously to e.g. 50 evaluations_to_spend=50, ) ``` If the run previously stopped due to reaching a budget and you specify the same budget, the worker will immediatly stop as it will remember the amount of budget it used previously. +!!! note "Auto-loading" + + When continuing a run, NePS automatically loads the search space and optimizer configuration from disk. You don't need to specify `pipeline_space=` or `optimizer=` again - NePS will use the saved settings from the original run. + +## Reconstructing and Reproducing Runs + +Sometimes you want to inspect what settings were used in a previous run, or reproduce a run with the same or modified settings. NePS provides utility functions to load both the search space and optimizer information: + +```python +import neps + +# Load everything from a previous run +root_dir = "path/to/previous_run" + +pipeline_space = neps.load_pipeline_space(root_dir) +optimizer_info = neps.load_optimizer_info(root_dir) + +print(f"Original optimizer: {optimizer_info['name']}") +print(f"Original search space: {pipeline_space}") + +# Option 1: Continue the original run (auto-loads everything) +neps.run( + evaluate_pipeline=my_function, + root_directory=root_dir, + evaluations_to_spend=100, # Increase budget +) + +# Option 2: Start a new run with the same settings +neps.run( + evaluate_pipeline=my_function, + pipeline_space=pipeline_space, + root_directory="path/to/new_run", + optimizer=optimizer_info['name'], + evaluations_to_spend=50, +) +``` + +For details on: + +- [`neps.load_pipeline_space()`][neps.api.load_pipeline_space] - see [Search Space Reference](neps_spaces.md#loading-the-search-space-from-disk) +- [`neps.load_optimizer_info()`][neps.api.load_optimizer_info] - see [Optimizer Reference](optimizers.md#24-loading-optimizer-information) + ## Overwriting a Run To overwrite a run, simply provide the same `root_directory=` to [`neps.run()`][neps.api.run] as before, with the `overwrite_root_directory=True` argument. @@ -140,7 +182,7 @@ provided to [`neps.run()`][neps.api.run]. │ └── config_2 │ ├── config.yaml │ └── metadata.json - ├── summary + ├── summary │ ├── full.csv │ └── short.csv │ ├── best_config_trajectory.txt diff --git a/docs/reference/neps_spaces.md b/docs/reference/neps_spaces.md new file mode 100644 index 000000000..3a32a7c6a --- /dev/null +++ b/docs/reference/neps_spaces.md @@ -0,0 +1,270 @@ +# NePS Spaces + +**NePS Spaces** provide a powerful framework for defining and optimizing complex search spaces across the entire pipeline, including [hyperparameters](#1-constructing-hyperparameter-spaces), [architecture search](#3-constructing-architecture-spaces) and [more](#4-constructing-complex-spaces). + +## 1. Constructing Hyperparameter Spaces + +**NePS spaces** include all the necessary components to define a Hyperparameter Optimization (HPO) search space like: + +- [`neps.Integer`][neps.space.neps_spaces.parameters.Integer]: Discrete integer values +- [`neps.Float`][neps.space.neps_spaces.parameters.Float]: Continuous float values +- [`neps.Categorical`][neps.space.neps_spaces.parameters.Categorical]: Discrete categorical values +- [`neps.IntegerFidelity`][neps.space.neps_spaces.parameters.IntegerFidelity]: Integer [multi-fidelity](../reference/search_algorithms/multifidelity.md) parameters (e.g., epochs, batch size) +- [`neps.FloatFidelity`][neps.space.neps_spaces.parameters.FloatFidelity]: Float [multi-fidelity](../reference/search_algorithms/multifidelity.md) parameters (e.g., dataset subset ratio) +- [`neps.Fidelity`][neps.space.neps_spaces.parameters.Fidelity]: Generic fidelity type (use IntegerFidelity or FloatFidelity instead) + +Using these types, you can define the parameters that NePS will optimize during the search process. +A **NePS space** is defined as a subclass of [`PipelineSpace`][neps.space.neps_spaces.parameters.PipelineSpace]. Here we define the hyperparameters that make up the space, like so: + +```python +import neps + +class MySpace(neps.PipelineSpace): + float_param = neps.Float(lower=0.1, upper=1.0) + int_param = neps.Integer(lower=1, upper=10) + cat_param = neps.Categorical(choices=("A", "B", "C")) +``` + +!!! info "Using **NePS Spaces**" + + To search a **NePS space**, pass it as the `pipeline_space` argument to the `neps.run()` function: + + ```python + neps.run( + ..., + pipeline_space=MySpace() + ) + ``` + + For more details on how to use the `neps.run()` function, see the [NePS Run Reference](../reference/neps_run.md). + +### Using cheap approximation, providing a [**Fidelity**](../reference/search_algorithms/landing_page_algo.md#what-is-multi-fidelity-optimization) Parameter + +You can use [`neps.IntegerFidelity`][neps.space.neps_spaces.parameters.IntegerFidelity] or [`neps.FloatFidelity`][neps.space.neps_spaces.parameters.FloatFidelity] to employ multi-fidelity optimization strategies, which can significantly speed up the optimization process by evaluating configurations at different fidelities (e.g., training for fewer epochs): + +```python +# Convenient syntax (recommended) +epochs = neps.IntegerFidelity(lower=1, upper=16) +subset_ratio = neps.FloatFidelity(lower=0.1, upper=1.0) + +# Alternative syntax (also works) +epochs = neps.IntegerFidelity(1, 16) +``` + +For more details on how to use fidelity parameters, see the [Multi-Fidelity](../reference/search_algorithms/landing_page_algo.md#what-is-multi-fidelity-optimization) section. + +### Using your knowledge, providing a [**Prior**](../reference/search_algorithms/landing_page_algo.md#what-are-priors) + +You can provide **your knowledge about where a good value for this parameter lies** by indicating a `prior=`. You can also specify a `prior_confidence=` to indicate how strongly you want NePS to focus on these, one of either `"low"`, `"medium"`, or `"high"`: + +```python +# Here "A" is used as a prior, indicated by its index 0 +cat_with_prior = neps.Categorical(choices=("A", "B", "C"), prior=0, prior_confidence="high") +``` + +For more details on how to use priors, see the [Priors](../reference/search_algorithms/landing_page_algo.md#what-are-priors) section. + +!!! info "Adding and removing parameters from **NePS Spaces**" + + To add or remove parameters from a `PipelineSpace` after its definition, you can use the `add()` and `remove()` methods. Mind you, these methods do NOT modify the existing space in-place, but return a new instance with the modifications: + + ```python + space = MySpace() + # Adding a new parameter using add() + larger_space = space.add(neps.Integer(lower=5, upper=15), name="new_int_param") + # Removing a parameter by its name + smaller_space = space.remove("cat_param") + ``` + +## 3. Constructing Architecture Spaces + +Additionally, **NePS spaces** can describe **complex (hierarchical) architectures** using: + +- [`Operation`][neps.space.neps_spaces.parameters.Operation]: Define operations and their arguments + +Operations can be Callables, (e.g. pytorch objects) which will be passed to the evaluation function as such: + +```python + +import torch.nn + +class NNSpace(PipelineSpace): + + # Defining operations for different activation functions + _relu = neps.Operation(operator=torch.nn.ReLU) + _sigmoid = neps.Operation(operator=torch.nn.Sigmoid) + + # We can then search over these operations and use them in the evaluation function + activation_function = neps.Categorical(choices=(_relu, _sigmoid)) +``` + +!!! info "Intermediate parameters" + + When defining parameters that should not be passed to the evaluation function and instead are used in other parameters, prefix them with an underscore, like here in `_layer_size`. Otherwise this might lead to `unexpected arguments` errors. + +Operation also allow for (keyword-)arguments to be defined, including other parameters of the space: + +```python + + batch_size = neps.Categorical(choices=(16, 32, 64)) + + _layer_size = neps.Integer(lower=80, upper=100) + + hidden_layer = neps.Operation( + operator=torch.nn.Linear, + kwargs={"input_size": 64, # Fixed input size + "output_size": _layer_size}, # Using the previously defined parameter + + # Or for non-keyword arguments: + args=(activation_function,) + ) +``` + +This can be used for efficient architecture search by defining cells and blocks of operations, that make up a neural network. +The `evaluate_pipeline` function will receive the sampled operations as Callables, which can be used to instantiate the model: + +```python +def evaluate_pipeline( + activation_function: torch.nn.Module, + batch_size: int, + hidden_layer: torch.nn.Linear): + + # Instantiate the model using the sampled operations + model = torch.nn.Sequential( + torch.nn.Flatten(), + hidden_layer, + activation_function, + torch.nn.Linear(in_features=hidden_layer.out_features, out_features=10) + ) + + # Use the model for training and return the validation accuracy + model.train(batch_size=batch_size, ...) + return model.evaluate(...).accuracy + +``` + +??? abstract "Structural Space-compatible optimizers" + + Currently, NePS Spaces is compatible with these optimizers, which can be imported from [neps.algorithms][neps.optimizers.algorithms--neps-algorithms]: + + - [`Random Search`][neps.optimizers.algorithms.random_search], which can sample the space uniformly at random + - [`Complex Random Search`][neps.optimizers.algorithms.complex_random_search], which can sample the space uniformly at random, using priors and mutating previously sampled configurations + - [`PriorBand`][neps.optimizers.algorithms.priorband], which uses [multi-fidelity](./search_algorithms/multifidelity.md) and the prior knowledge encoded in the NePS space + +## 4. Constructing Complex Spaces + +Until now all parameters are sampled once and their value used for all occurrences. This section describes how to resample parameters in different contexts using: + +- [`.resample()`][neps.space.neps_spaces.parameters.Resample]: Resample from an existing parameters range + +With `.resample()` you can reuse a parameter, even themselves recursively, but with a new value each time: + +```python +class ResampleSpace(neps.PipelineSpace): + float_param = neps.Float(lower=0, upper=1) + + # The resampled parameter will have the same range but will be sampled + # independently, so it can take a different value than its source + resampled_float = float_param.resample() +``` + +This is especially useful for defining complex architectures, where e.g. a cell block is defined and then resampled multiple times to create a neural network architecture: + +```python +class CNN_Space(neps.PipelineSpace): + _kernel_size = neps.Integer(lower=5, upper=8) + + # Define a cell block that can be resampled + # It will resample a new kernel size from _kernel_size each time + # Each instance will be identically but independently sampled + _cell_block = neps.Operation( + operator=torch.nn.Conv2d, + kwargs={"kernel_size": _kernel_size.resample()} + ) + + # Resample the cell block multiple times to create a convolutional neural network + cnn = torch.nn.Sequential( + _cell_block.resample(), + _cell_block.resample(), + _cell_block.resample(), + ) + +def evaluate_pipeline(cnn: torch.nn.Module): + # Use the cnn model for training and return the validation accuracy + cnn.train(...) + return cnn.evaluate(...).accuracy +``` + +??? info "Self- and future references" + + When referencing itself or a not yet defined parameter (to enable recursions) use [`neps.ByName`][neps.space.neps_spaces.parameters.ByName] to reference the parameter by its string name: + + ```python + self_reference = Categorical( + choices=( + # It will either choose to resample itself twice + (neps.ByName("self_reference").resample(), neps.ByName("self_reference").resample()), + # Or it will sample the future parameter + (neps.ByName("future_param").resample(),), + ) + ) + # This results in a (possibly infinite) tuple of independently sampled future_params + + future_param = Float(lower=0, upper=5) + ``` + +!!! tip "Complex structural spaces" + + Together, [Resampling][neps.space.neps_spaces.parameters.Resample] and [operations][neps.space.neps_spaces.parameters.Operation] allow you to define complex search spaces across the whole ML-pipeline akin to [Context-Free Grammars (CFGs)](https://en.wikipedia.org/wiki/Context-free_grammar), exceeding architecture search. For example, you can sample neural optimizers from a set of instructions, as done in [`NOSBench`](https://openreview.net/pdf?id=5Lm2ghxMlp) to train models. + +## Inspecting Configurations + +NePS saves the configurations as paths, where each sampling decision is recorded. As they are hard to read, so you can load the configuration using `neps.load_config()`, which returns a dictionary with the resolved parameters and their values: + +```python +import neps + +pipeline = neps.load_config("Path/to/config.yaml", pipeline_space=SimpleSpace()) # or +pipeline = neps.load_config("Path/to/neps_folder", config_id="config_0", pipeline_space=SimpleSpace()) + +# The pipeline now contains all the parameters and their values the same way they would be given to the evaluate_pipeline, e.g. the callable model: +model = pipeline["model"] +``` + +### Loading the Search Space from Disk + +NePS automatically saves the search space when you run an optimization. You can retrieve it later using `neps.load_pipeline_space()`: + +```python +import neps + +# Load the search space from a previous run +pipeline_space = neps.load_pipeline_space("Path/to/neps_folder") + +# Now you can use it to inspect configurations, continue runs, or analysis +``` + +!!! note "Auto-loading" + + In most cases, you don't need to call `load_pipeline_space()` explicitly. When continuing a run, `neps.run()` automatically loads the search space from disk. See [Continuing Runs](neps_run.md#continuing-runs) for more details. + +!!! tip "Reconstructing a Run" + + You can load both the search space and optimizer information to fully reconstruct a previous run. See [Reconstructing and Reproducing Runs](neps_run.md#reconstructing-and-reproducing-runs) for a complete example. + +## Using ConfigSpace + +For users familiar with the [`ConfigSpace`](https://automl.github.io/ConfigSpace/main/) library, +can also define the `pipeline_space` through `ConfigurationSpace()` + +```python +from configspace import ConfigurationSpace, Float + +configspace = ConfigurationSpace( + { + "learning_rate": Float("learning_rate", bounds=(1e-4, 1e-1), log=True) + "optimizer": ["adam", "sgd", "rmsprop"], + "dropout_rate": 0.5, + } +) +``` diff --git a/docs/reference/optimizers.md b/docs/reference/optimizers.md index 6edef593a..61fb48b78 100644 --- a/docs/reference/optimizers.md +++ b/docs/reference/optimizers.md @@ -42,18 +42,19 @@ NePS provides a multitude of optimizers from the literature, the [algorithms](.. ✅ = supported/necessary, ❌ = not supported, ✔️* = optional, click for details, ✖️\* ignorable, click for details -| Algorithm | [Multi-Fidelity](../reference/search_algorithms/multifidelity.md) | [Priors](../reference/search_algorithms/prior.md) | Model-based | -| :- | :------------: | :----: | :---------: | -| `Grid Search`|[️️✖️*][neps.optimizers.algorithms.grid_search]|❌|❌| -| `Random Search`|[️️✖️*][neps.optimizers.algorithms.random_search]|[✔️*][neps.optimizers.algorithms.random_search]|❌| -| [`Bayesian Optimization`](../reference/search_algorithms/bayesian_optimization.md)|[️️✖️*][neps.optimizers.algorithms.bayesian_optimization]|❌|✅| -| [`Successive Halving`](../reference/search_algorithms/multifidelity.md#1-successive-halfing)|✅|[✔️*][neps.optimizers.algorithms.successive_halving]|❌| -| [`ASHA`](../reference/search_algorithms/multifidelity.md#asynchronous-successive-halving)|✅|[✔️*][neps.optimizers.algorithms.asha]|❌| -| [`Hyperband`](../reference/search_algorithms/multifidelity.md#2-hyperband)|✅|[✔️*][neps.optimizers.algorithms.hyperband]|❌| -| [`Asynch HB`](../reference/search_algorithms/multifidelity.md)|✅|[✔️*][neps.optimizers.algorithms.async_hb]|❌| -| [`IfBO`](../reference/search_algorithms/multifidelity.md#3-in-context-freeze-thaw-bayesian-optimization)|✅|[✔️*][neps.optimizers.algorithms.ifbo]|✅| -| [`PiBO`](../reference/search_algorithms/prior.md#1-pibo)|[️️✖️*][neps.optimizers.algorithms.pibo]|✅|✅| -| [`PriorBand`](../reference/search_algorithms/multifidelity_prior.md#1-priorband)|✅|✅|✅| +| Algorithm | [Multi-Fidelity](../reference/search_algorithms/multifidelity.md) | [Priors](../reference/search_algorithms/prior.md) | Model-based | [NePS-ready](../reference/neps_spaces.md#3-constructing-architecture-spaces) | +| :- | :------------: | :----: | :---------: | :-----------------: | +| `Grid Search`|[️️✖️*][neps.optimizers.algorithms.grid_search]|❌|❌|❌| +| `Random Search`|[️️✖️*][neps.optimizers.algorithms.random_search]|[✔️*][neps.optimizers.algorithms.random_search]|❌|✅| +| `Complex Random Search`|[️️✖️*][neps.optimizers.algorithms.complex_random_search]|[✔️*][neps.optimizers.algorithms.complex_random_search]|❌|✅| +| [`Bayesian Optimization`](../reference/search_algorithms/bayesian_optimization.md)|[️️✖️*][neps.optimizers.algorithms.bayesian_optimization]|❌|✅|❌| +| [`Successive Halving`](../reference/search_algorithms/multifidelity.md#1-successive-halfing)|✅|[✔️*][neps.optimizers.algorithms.successive_halving]|❌|❌| +| [`ASHA`](../reference/search_algorithms/multifidelity.md#asynchronous-successive-halving)|✅|[✔️*][neps.optimizers.algorithms.asha]|❌|❌| +| [`Hyperband`](../reference/search_algorithms/multifidelity.md#2-hyperband)|✅|[✔️*][neps.optimizers.algorithms.hyperband]|❌|❌| +| [`Asynch HB`](../reference/search_algorithms/multifidelity.md)|✅|[✔️*][neps.optimizers.algorithms.async_hb]|❌|❌| +| [`IfBO`](../reference/search_algorithms/multifidelity.md#3-in-context-freeze-thaw-bayesian-optimization)|✅|[✔️*][neps.optimizers.algorithms.ifbo]|✅|❌| +| [`PiBO`](../reference/search_algorithms/prior.md#1-pibo)|[️️✖️*][neps.optimizers.algorithms.pibo]|✅|✅|❌| +| [`PriorBand`](../reference/search_algorithms/multifidelity_prior.md#1-priorband)|✅|✅|✅|✅| If you prefer not to specify a particular optimizer for your AutoML task, you can simply pass `"auto"` or `None` for the neps optimizer. This provides a hassle-free way to get started quickly, as NePS will automatically choose the best optimizer based on the characteristics of your search @@ -109,6 +110,31 @@ neps.run( ) ``` +### 2.4 Loading Optimizer Information + +NePS automatically saves the optimizer metadata (name and configuration) when you run an optimization. You can retrieve this information later using `neps.load_optimizer_info()`: + +```python +import neps + +# Load the optimizer info from a previous run +optimizer_info = neps.load_optimizer_info("path/to/neps_folder") + +# Access the optimizer name and configuration +print(f"Optimizer: {optimizer_info['name']}") +print(f"Configuration: {optimizer_info['info']}") +``` + +This is useful for: + +- **Inspecting** what optimizer and settings were used in a previous run +- **Reproducing** experiments with the same optimizer configuration +- **Comparing** different optimizer settings across runs + +!!! tip "Reconstructing a Complete Run" + + Combine `load_optimizer_info()` with `load_pipeline_space()` to fully reconstruct a previous optimization. See [Reconstructing and Reproducing Runs](neps_run.md#reconstructing-and-reproducing-runs) for a complete example. + ## 3 Custom Optimizers To design entirely new optimizers, you can define them as class with a `__call__` method outside of NePS and pass them to the `neps.run()` function: diff --git a/docs/reference/pipeline_space.md b/docs/reference/pipeline_space.md deleted file mode 100644 index 9844e42a3..000000000 --- a/docs/reference/pipeline_space.md +++ /dev/null @@ -1,108 +0,0 @@ -# Initializing the Pipeline Space - -In NePS, we need to define a `pipeline_space`. -This space can be structured through various approaches, including a Python dictionary, or ConfigSpace. -Each of these methods allows you to specify a set of parameter types, ranging from Float and Categorical to specialized architecture parameters. -Whether you choose a dictionary, or ConfigSpace, your selected method serves as a container or framework -within which these parameters are defined and organized. This section not only guides you through the process of -setting up your `pipeline_space` using these methods but also provides detailed instructions and examples on how to -effectively incorporate various parameter types, ensuring that NePS can utilize them in the optimization process. - - -## Parameters -NePS currently features 4 primary hyperparameter types: - -* [`Categorical`][neps.space.Categorical] -* [`Float`][neps.space.Float] -* [`Integer`][neps.space.Integer] -* [`Constant`][neps.space.Constant] - -Using these types, you can define the parameters that NePS will optimize during the search process. -The most basic way to pass these parameters is through a Python dictionary, where each key-value -pair represents a parameter name and its respective type. -For example, the following Python dictionary defines a `pipeline_space` with four parameters -for optimizing a deep learning model: - -```python -pipeline_space = { - "learning_rate": neps.Float(0.00001, 0.1, log=True), - "num_epochs": neps.Integer(3, 30, is_fidelity=True), - "optimizer": ["adam", "sgd", "rmsprop"], # Categorical - "dropout_rate": 0.5, # Constant -} - -neps.run(.., pipeline_space=pipeline_space) -``` - -??? example "Quick Parameter Reference" - - === "`Categorical`" - - ::: neps.space.Categorical - - === "`Float`" - - ::: neps.space.Float - - === "`Integer`" - - ::: neps.space.Integer - - === "`Constant`" - - ::: neps.space.Constant - - -## Using your knowledge, providing a Prior -When optimizing, you can provide your own knowledge using the parameter `prior=`. -By indicating a `prior=` we take this to be your user prior, -**your knowledge about where a good value for this parameter lies**. - -You can also specify a `prior_confidence=` to indicate how strongly you want NePS, -to focus on these, one of either `"low"`, `"medium"`, or `"high"`. - -```python -import neps - -neps.run( - ..., - pipeline_space={ - "learning_rate": neps.Float(1e-4, 1e-1, log=True, prior=1e-2, prior_confidence="medium"), - "num_epochs": neps.Integer(3, 30, is_fidelity=True), - "optimizer": neps.Categorical(["adam", "sgd", "rmsprop"], prior="adam", prior_confidence="low"), - "dropout_rate": neps.Constant(0.5), - } -) -``` - -!!! warning "Interaction with `is_fidelity`" - - If you specify `is_fidelity=True` and `prior=` for one parameter, this will raise an error. - -Currently the two major algorithms that exploit this in NePS are `PriorBand` -(prior-based `HyperBand`) and `PiBO`, a version of Bayesian Optimization which uses Priors. For more information on priors and algorithms using them, please refer to the [prior documentation](../reference/search_algorithms/prior.md). - -## Using ConfigSpace - -For users familiar with the [`ConfigSpace`](https://automl.github.io/ConfigSpace/main/) library, -can also define the `pipeline_space` through `ConfigurationSpace()` - -```python -from configspace import ConfigurationSpace, Float - -configspace = ConfigurationSpace( - { - "learning_rate": Float("learning_rate", bounds=(1e-4, 1e-1), log=True) - "optimizer": ["adam", "sgd", "rmsprop"], - "dropout_rate": 0.5, - } -) -``` - -!!! warning - - Parameters you wish to use as a **fidelity** are not support through ConfigSpace - at this time. - -For additional information on ConfigSpace and its features, please visit the following -[link](https://github.com/automl/ConfigSpace). diff --git a/docs/reference/search_algorithms/landing_page_algo.md b/docs/reference/search_algorithms/landing_page_algo.md index 7f7be891e..5d818d519 100644 --- a/docs/reference/search_algorithms/landing_page_algo.md +++ b/docs/reference/search_algorithms/landing_page_algo.md @@ -6,18 +6,19 @@ We distinguish between algorithms that use different types of information and st ✅ = supported/necessary, ❌ = not supported, ✔️* = optional, click for details, ✖️\* ignorable, click for details -| Algorithm | [Multi-Fidelity](../search_algorithms/multifidelity.md) | [Priors](../search_algorithms/prior.md) | Model-based | -| :- | :------------: | :----: | :---------: | -| `Grid Search`|[️️✖️*][neps.optimizers.algorithms.grid_search]|❌|❌| -| `Random Search`|[️️✖️*][neps.optimizers.algorithms.random_search]|[✔️*][neps.optimizers.algorithms.random_search]|❌| -| [`Bayesian Optimization`](../search_algorithms/bayesian_optimization.md)|[️️✖️*][neps.optimizers.algorithms.bayesian_optimization]|❌|✅| -| [`Successive Halving`](../search_algorithms/multifidelity.md#1-successive-halfing)|✅|[✔️*][neps.optimizers.algorithms.successive_halving]|❌| -| [`ASHA`](../search_algorithms/multifidelity.md#asynchronous-successive-halving)|✅|[✔️*][neps.optimizers.algorithms.asha]|❌| -| [`Hyperband`](../search_algorithms/multifidelity.md#2-hyperband)|✅|[✔️*][neps.optimizers.algorithms.hyperband]|❌| -| [`Asynch HB`](../search_algorithms/multifidelity.md)|✅|[✔️*][neps.optimizers.algorithms.async_hb]|❌| -| [`IfBO`](../search_algorithms/multifidelity.md#3-in-context-freeze-thaw-bayesian-optimization)|✅|[✔️*][neps.optimizers.algorithms.ifbo]|✅| -| [`PiBO`](../search_algorithms/prior.md#1-pibo)|[️️✖️*][neps.optimizers.algorithms.pibo]|✅|✅| -| [`PriorBand`](../search_algorithms/multifidelity_prior.md#1-priorband)|✅|✅|✅| +| Algorithm | [Multi-Fidelity](../search_algorithms/multifidelity.md) | [Priors](../search_algorithms/prior.md) | Model-based | [NePS-ready](../neps_spaces.md#3-constructing-architecture-spaces) | +| :- | :------------: | :----: | :---------: | :-----------------: | +| `Grid Search`|[️️✖️*][neps.optimizers.algorithms.grid_search]|❌|❌|❌| +| `Random Search`|[️️✖️*][neps.optimizers.algorithms.random_search]|[✔️*][neps.optimizers.algorithms.random_search]|❌|✅| +| `Complex Random Search`|[️️✖️*][neps.optimizers.algorithms.complex_random_search]|[✔️*][neps.optimizers.algorithms.complex_random_search]|❌|✅| +| [`Bayesian Optimization`](../search_algorithms/bayesian_optimization.md)|[️️✖️*][neps.optimizers.algorithms.bayesian_optimization]|❌|✅|❌| +| [`Successive Halving`](../search_algorithms/multifidelity.md#1-successive-halfing)|✅|[✔️*][neps.optimizers.algorithms.successive_halving]|❌|❌| +| [`ASHA`](../search_algorithms/multifidelity.md#asynchronous-successive-halving)|✅|[✔️*][neps.optimizers.algorithms.asha]|❌|❌| +| [`Hyperband`](../search_algorithms/multifidelity.md#2-hyperband)|✅|[✔️*][neps.optimizers.algorithms.hyperband]|❌|❌| +| [`Asynch HB`](../search_algorithms/multifidelity.md)|✅|[✔️*][neps.optimizers.algorithms.async_hb]|❌|❌| +| [`IfBO`](../search_algorithms/multifidelity.md#3-in-context-freeze-thaw-bayesian-optimization)|✅|[✔️*][neps.optimizers.algorithms.ifbo]|✅|❌| +| [`PiBO`](../search_algorithms/prior.md#1-pibo)|[️️✖️*][neps.optimizers.algorithms.pibo]|✅|✅|❌| +| [`PriorBand`](../search_algorithms/multifidelity_prior.md#1-priorband)|✅|✅|✅|✅| ## What is Multi-Fidelity Optimization? @@ -36,7 +37,7 @@ We present a collection of MF-algorithms [here](./multifidelity.md) and algorith ## What are Priors? -Priors are used when there exists some information about the search space, that can be used to guide the optimization process. This information could come from expert domain knowledge or previous experiments. A Prior is provided in the form of a distribution over one dimension of the search space, with a `mean` (the suspected optimum) and a `confidence level`, or `variance`. We discuss how Priors can be included in your NePS-search space [here](../../reference/pipeline_space.md#using-your-knowledge-providing-a-prior). +Priors are used when there exists some information about the search space, that can be used to guide the optimization process. This information could come from expert domain knowledge or previous experiments. A Prior is provided in the form of a distribution over one dimension of the search space, with a `mean` (the suspected optimum) and a `confidence level`, or `variance`. We discuss how Priors can be included in your NePS-search space [here](../../reference/neps_spaces.md#1-constructing-hyperparameter-spaces). !!! tip "Advantages of using Priors" diff --git a/docs/reference/search_algorithms/multifidelity.md b/docs/reference/search_algorithms/multifidelity.md index 171b1cc41..c9accfb01 100644 --- a/docs/reference/search_algorithms/multifidelity.md +++ b/docs/reference/search_algorithms/multifidelity.md @@ -108,7 +108,7 @@ See the algorithm's implementation details in the [api][neps.optimizers.algorith ??? example "Practical Tips" - - ``IfBO`` is a good choice when the problem allows for low-fidelity configurations to be continued to retrieve high-fidelity results, utilizing neps's [checkpointing](../evaluate_pipeline.md#arguments-for-convenience) feature. + - ``IfBO`` is a good choice when the problem allows for low-fidelity configurations to be continued to retrieve high-fidelity results, utilizing neps's [checkpointing](../evaluate_pipeline.md#4-extra-injected-arguments) feature. ___ For optimizers using both Priors and Multi-Fidelity, please refer [here](multifidelity_prior.md). diff --git a/mkdocs.yml b/mkdocs.yml index f10fe23b8..1218edc6a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -139,7 +139,7 @@ nav: - Getting Started: 'getting_started.md' - Reference: - Run: 'reference/neps_run.md' - - Search Space: 'reference/pipeline_space.md' + - NePS Spaces: 'reference/neps_spaces.md' - The Evaluate Function: 'reference/evaluate_pipeline.md' - Analysing Runs: 'reference/analyse.md' - Optimizer: 'reference/optimizers.md' diff --git a/neps/__init__.py b/neps/__init__.py index af54987b3..f99c46caa 100644 --- a/neps/__init__.py +++ b/neps/__init__.py @@ -1,29 +1,74 @@ -from neps.api import import_trials, run, save_pipeline_results +"""NePS: A framework for Neural Architecture Search and Hyperparameter Optimization. +This module provides a unified interface for defining search spaces, running optimizers, +and visualizing results. It includes various optimizers, search space definitions, +and plotting utilities, making it easy to experiment with different configurations +and algorithms. +""" + +from neps.api import ( + create_config, + import_trials, + load_config, + load_optimizer_info, + load_pipeline_space, + run, + save_pipeline_results, +) from neps.optimizers import algorithms from neps.optimizers.ask_and_tell import AskAndTell from neps.optimizers.optimizer import SampledConfig from neps.plot.plot import plot from neps.plot.tensorboard_eval import tblogger -from neps.space import Categorical, Constant, Float, Integer, SearchSpace +from neps.space import HPOCategorical, HPOConstant, HPOFloat, HPOInteger, SearchSpace +from neps.space.neps_spaces.parameters import ( + ByName, + Categorical, + ConfidenceLevel, + Fidelity, + Float, + FloatFidelity, + Integer, + IntegerFidelity, + Operation, + PipelineSpace, + Resample, +) from neps.state import BudgetInfo, Trial from neps.state.pipeline_eval import UserResultDict from neps.status.status import status -from neps.utils.files import load_and_merge_yamls as load_yamls +from neps.utils import convert_operation_to_callable +from neps.utils.files import load_and_merge_yamls __all__ = [ "AskAndTell", "BudgetInfo", + "ByName", "Categorical", - "Constant", + "ConfidenceLevel", + "Fidelity", "Float", + "FloatFidelity", + "HPOCategorical", + "HPOConstant", + "HPOFloat", + "HPOInteger", "Integer", + "IntegerFidelity", + "Operation", + "PipelineSpace", + "Resample", "SampledConfig", "SearchSpace", "Trial", "UserResultDict", "algorithms", + "convert_operation_to_callable", + "create_config", "import_trials", - "load_yamls", + "load_and_merge_yamls", + "load_config", + "load_optimizer_info", + "load_pipeline_space", "plot", "run", "save_pipeline_results", diff --git a/neps/api.py b/neps/api.py index da20f098f..991fc6faf 100644 --- a/neps/api.py +++ b/neps/api.py @@ -3,17 +3,31 @@ from __future__ import annotations import logging +import shutil import warnings from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, Concatenate, Literal +import yaml + from neps.normalization import _normalize_imported_config -from neps.optimizers import AskFunction, OptimizerChoice, load_optimizer +from neps.optimizers import AskFunction, OptimizerChoice, OptimizerInfo, load_optimizer from neps.runtime import _launch_runtime, _save_results +from neps.space import SearchSpace +from neps.space.neps_spaces.neps_space import ( + adjust_evaluation_pipeline_for_neps_space, + check_neps_space_compatibility, + convert_classic_to_neps_search_space, + convert_neps_to_classic_search_space, + convert_operation_to_callable, + resolve, +) +from neps.space.neps_spaces.parameters import Operation, PipelineSpace +from neps.space.neps_spaces.string_formatter import format_value from neps.space.parsing import convert_to_space from neps.state import NePSState, OptimizationState, SeedSnapshot -from neps.status.status import post_run_csv, trajectory_of_improvements +from neps.status.status import post_run_csv from neps.utils.common import dynamic_load_object from neps.validation import _validate_imported_config, _validate_imported_result @@ -21,25 +35,19 @@ from ConfigSpace import ConfigurationSpace from neps.optimizers.algorithms import CustomOptimizer - from neps.space import Parameter, SearchSpace - from neps.state import EvaluatePipelineReturn - from neps.state.pipeline_eval import UserResultDict + from neps.state.pipeline_eval import EvaluatePipelineReturn, UserResultDict logger = logging.getLogger(__name__) -def run( # noqa: D417, PLR0913 +def run( # noqa: C901, D417, PLR0912, PLR0913, PLR0915 evaluate_pipeline: Callable[..., EvaluatePipelineReturn] | str, - pipeline_space: ( - Mapping[str, dict | str | int | float | Parameter] - | SearchSpace - | ConfigurationSpace - ), + pipeline_space: ConfigurationSpace | PipelineSpace | SearchSpace | dict | None = None, *, root_directory: str | Path = "neps_results", overwrite_root_directory: bool = False, evaluations_to_spend: int | None = None, - max_evaluations_per_run: int | None = None, + max_evaluations_per_run: int | None = None, # deprecated continue_until_max_evaluation_completed: bool = False, cost_to_spend: int | float | None = None, fidelities_to_spend: int | float | None = None, @@ -52,7 +60,9 @@ def run( # noqa: D417, PLR0913 OptimizerChoice | Mapping[str, Any] | tuple[OptimizerChoice, Mapping[str, Any]] - | Callable[Concatenate[SearchSpace, ...], AskFunction] + | Callable[Concatenate[SearchSpace, ...], AskFunction] # Hack, while we transit + | Callable[Concatenate[PipelineSpace, ...], AskFunction] # from SearchSpace to + | Callable[Concatenate[SearchSpace | PipelineSpace, ...], AskFunction] # Pipeline | CustomOptimizer | Literal["auto"] ) = "auto", @@ -76,30 +86,28 @@ def evaluate_pipeline(some_parameter: float) -> float: validation_error = -some_parameter return validation_error - pipeline_space = dict(some_parameter=neps.Float(lower=0, upper=1)) + class MySpace(PipelineSpace): + dataset = "mnist" # constant + nlayers = neps.Integer(2,10) # integer + alpha = neps.Float(0.1, 1.0) # float + optimizer = neps.Categorical( # categorical + ("adam", "sgd", "rmsprop") + ) + learning_rate = neps.Float( # log spaced float + lower=1e-5, upper=1, log=True + ) + epochs = # fidelity integer + neps.IntegerFidelity(1, 100) + batch_size = neps.Integer( # integer with a prior + lower=32, + upper=512, + prior=128, + prior_confidence="medium" + ) + neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space={ - "some_parameter": (0.0, 1.0), # float - "another_parameter": (0, 10), # integer - "optimizer": ["sgd", "adam"], # categorical - "epoch": neps.Integer( # fidelity integer - lower=1, - upper=100, - is_fidelity=True - ), - "learning_rate": neps.Float( # log spaced float - lower=1e-5, - upper=1, - log=True - ), - "alpha": neps.Float( # float with a prior - lower=0.1, - upper=1.0, - prior=0.99, - prior_confidence="high", - ) - }, + pipeline_space=MySpace(), root_directory="usage_example", evaluations_to_spend=5, ) @@ -128,30 +136,40 @@ def evaluate_pipeline(some_parameter: float) -> float: to specify the function to call. You may also directly provide an mode to import, e.g., `"my.module.something:evaluate_pipeline"`. - pipeline_space: The search space to minimize over. + pipeline_space: The pipeline space to minimize over. - This most direct way to specify the search space is as follows: + !!! tip "Optional for continuing runs" + + This parameter is **required** for the first run but **optional** when + continuing an existing optimization. If not provided, NePS will + automatically load the pipeline space from `root_directory/pipeline_space.pkl`. + + When provided for a continuing run, NePS will validate that it matches + the one saved on disk to prevent inconsistencies. + + This most direct way to specify the pipeline space is as follows: ```python - neps.run( - pipeline_space={ - "dataset": "mnist", # constant - "nlayers": (2, 10), # integer - "alpha": (0.1, 1.0), # float - "optimizer": [ # categorical - "adam", "sgd", "rmsprop" - ], - "learning_rate": neps.Float(, # log spaced float - lower=1e-5, upper=1, log=True - ), - "epochs": neps.Integer( # fidelity integer - lower=1, upper=100, is_fidelity=True - ), - "batch_size": neps.Integer( # integer with a prior - lower=32, upper=512, prior=128 - ), + class MySpace(PipelineSpace): + nlayers = neps.Integer(2,10) # integer + alpha = neps.Float(0.1, 1.0) # float + optimizer = neps.Categorical( # categorical + ("adam", "sgd", "rmsprop") + ) + learning_rate = neps.Float( # log spaced float + lower=1e-5, upper=1, log=True + ) + epochs = # fidelity integer + neps.IntegerFidelity(1, 100) + batch_size = neps.Integer( # integer with a prior + lower=32, + upper=512, + prior=128, + prior_confidence="medium" + ) - } + neps.run( + pipeline_space=MySpace() ) ``` @@ -163,29 +181,8 @@ def evaluate_pipeline(some_parameter: float) -> float: * `prior=`: If you have a good idea about what a good setting for a parameter may be, you can set this as the prior for - a parameter. You can specify this along with `prior_confidence` - if you would like to assign a `"low"`, `"medium"`, or `"high"` - confidence to the prior. - - - !!! note "Yaml support" - - To support spaces defined in yaml, you may also define the parameters - as dictionarys, e.g., - - ```python - neps.run( - pipeline_space={ - "dataset": "mnist", - "nlayers": {"type": "int", "lower": 2, "upper": 10}, - "alpha": {"type": "float", "lower": 0.1, "upper": 1.0}, - "optimizer": {"type": "cat", "choices": ["adam", "sgd", "rmsprop"]}, - "learning_rate": {"type": "float", "lower": 1e-5, "upper": 1, "log": True}, - "epochs": {"type": "int", "lower": 1, "upper": 100, "is_fidelity": True}, - "batch_size": {"type": "int", "lower": 32, "upper": 512, "prior": 128}, - } - ) - ``` + a parameter. You specify this along with `prior_confidence` + to assign a `"low"`, `"medium"`, or `"high"`confidence to the prior. !!! note "ConfigSpace support" @@ -198,10 +195,10 @@ def evaluate_pipeline(some_parameter: float) -> float: the run. This is, e.g., useful when debugging a evaluate_pipeline function. evaluations_to_spend: Number of evaluations this specific call/worker should do. - ??? note "Limitation on Async mode" - Currently, there is no specific number to control number of parallel evaluations running with - the same worker, so in case you want to limit the number of parallel evaluations, - it's crucial to limit the `evaluations_to_spend` accordingly. + ??? note "Limitation on Async mode" + Currently, there is no specific number to control number of parallel evaluations running with + the same worker, so in case you want to limit the number of parallel evaluations, + it's crucial to limit the `evaluations_to_spend` accordingly. continue_until_max_evaluation_completed: If true, stop only after evaluations_to_spend have fully completed. In other words, @@ -268,102 +265,11 @@ def evaluate_pipeline(some_parameter: float) -> float: optimizer: Which optimizer to use. Not sure which to use? Leave this at `"auto"` and neps will - choose the optimizer based on the search space given. + choose the optimizer based on the pipeline space given. ??? note "Available optimizers" - --- - - * `#!python "bayesian_optimization"`, - - ::: neps.optimizers.algorithms.bayesian_optimization - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "ifbo"` - - ::: neps.optimizers.algorithms.ifbo - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "successive_halving"`: - - ::: neps.optimizers.algorithms.successive_halving - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "hyperband"`: - - ::: neps.optimizers.algorithms.hyperband - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "priorband"`: - - ::: neps.optimizers.algorithms.priorband - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "asha"`: - - ::: neps.optimizers.algorithms.asha - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "async_hb"`: - - ::: neps.optimizers.algorithms.async_hb - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "random_search"`: - - ::: neps.optimizers.algorithms.random_search - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - - * `#!python "grid_search"`: - - ::: neps.optimizers.algorithms.grid_search - options: - show_root_heading: false - show_signature: false - show_source: false - - --- - + See the [optimizers documentation](../../reference/search_algorithms/landing_page_algo.md) for a list of available optimizers. With any optimizer choice, you also may provide some additional parameters to the optimizers. We do not recommend this unless you are familiar with the optimizer you are using. You @@ -373,10 +279,11 @@ def evaluate_pipeline(some_parameter: float) -> float: ```python neps.run( ..., - optimzier={ - "name": "priorband", - "sample_prior_first": True, - } + optimizer=("priorband", + { + "sample_prior_first": True, + } + ) ) ``` @@ -423,6 +330,45 @@ def __call__( "`evaluations_to_spend` for limiting the number of evaluations for this run.", ) + # If the pipeline_space is a SearchSpace, convert it to a PipelineSpace and throw a + # deprecation warning + if isinstance(pipeline_space, SearchSpace | dict): + if isinstance(pipeline_space, dict): + pipeline_space = SearchSpace(pipeline_space) + pipeline_space = convert_classic_to_neps_search_space(pipeline_space) + space_lines = str(pipeline_space).split("\n") + space_def = space_lines[1] if len(space_lines) > 1 else str(pipeline_space) + warnings.warn( + "Passing a SearchSpace or dictionary to neps.run is deprecated and will be" + " removed in a future version. Please pass a PipelineSpace instead, as" + " described in the NePS-Spaces documentation." + " This specific space should be given as:\n\n```python\nclass" + f" MySpace(PipelineSpace):\n{space_def}\n```\n", + DeprecationWarning, + stacklevel=2, + ) + + # Try to load pipeline_space from disk if not provided + if pipeline_space is None: + root_path = Path(root_directory) + if root_path.exists() and not overwrite_root_directory: + try: + pipeline_space = load_pipeline_space(root_path) + logger.info( + "Loaded pipeline space from disk. Continuing optimization with " + f"existing pipeline space from {root_path}" + ) + except (FileNotFoundError, ValueError) as e: + # If loading fails, we'll error below + logger.debug(f"Could not load pipeline space from disk: {e}") + + # If still None, raise error + if pipeline_space is None: + raise ValueError( + "pipeline_space is required for the first run. For continuing an" + " existing run, the pipeline space will be loaded from disk. No existing" + f" pipeline space found at: {root_directory}" + ) controling_params = { "evaluations_to_spend": evaluations_to_spend, "cost_to_spend": cost_to_spend, @@ -438,28 +384,65 @@ def __call__( ) logger.info(f"Starting neps.run using root directory {root_directory}") - space = convert_to_space(pipeline_space) - _optimizer_ask, _optimizer_info = load_optimizer(optimizer=optimizer, space=space) - multi_fidelity_optimizers = { - "successive_halving", - "asha", - "hyperband", - "async_hb", - "ifbo", - "priorband", - "moasha", - "mo_hyperband", - "primo", - } + # Check if we're continuing an existing run and should load the optimizer from disk + root_path = Path(root_directory) + optimizer_info_path = root_path / "optimizer_info.yaml" + is_continuing_run = optimizer_info_path.exists() and not overwrite_root_directory + + # If continuing a run and optimizer is "auto" (default), load existing optimizer + # with its parameters + if is_continuing_run and optimizer == "auto": + try: + existing_optimizer_info = load_optimizer_info(root_path) + logger.info( + "Continuing optimization with existing optimizer: " + f"{existing_optimizer_info['name']}" + ) + # Use the existing optimizer with its original parameters + optimizer = ( + existing_optimizer_info["name"], + existing_optimizer_info["info"], + ) # type: ignore + except (FileNotFoundError, KeyError) as e: + # No existing optimizer found or invalid format, proceed with auto + logger.debug(f"Could not load existing optimizer info: {e}") + + # Check if the pipeline_space only contains basic HPO parameters. + # If yes, we convert it to a classic SearchSpace, to use with the old optimizers. + # If no, we use adjust_evaluation_pipeline_for_neps_space to convert the + # pipeline_space and only use the new NEPS optimizers. + + # If the optimizer is not a NEPS algorithm, we try to convert the pipeline_space + + neps_classic_space_compatibility = check_neps_space_compatibility(optimizer) + if neps_classic_space_compatibility in ["both", "classic"] and isinstance( + pipeline_space, PipelineSpace + ): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space: + pipeline_space = converted_space + space = convert_to_space(pipeline_space) - is_multi_fidelity = _optimizer_info["name"] in multi_fidelity_optimizers + if neps_classic_space_compatibility == "neps" and not isinstance( + space, PipelineSpace + ): + space = convert_classic_to_neps_search_space(space) - if not is_multi_fidelity and fidelities_to_spend is not None: + # Optimizer check, if the search space is a Pipeline and the optimizer is not a NEPS + # algorithm, we raise an error, as the optimizer is not compatible. + if isinstance(space, PipelineSpace) and neps_classic_space_compatibility == "classic": raise ValueError( - "`fidelities_to_spend` is not allowed for non-multi-fidelity optimizers." + f"The provided optimizer {optimizer} is not compatible with this complex" + " search space. Please use one that is, such as 'random_search'," + " 'hyperband', 'priorband', or 'complex_random_search'." ) + # Log the search space after conversion + logger.info(str(space)) + + _optimizer_ask, _optimizer_info = load_optimizer(optimizer=optimizer, space=space) + _eval: Callable if isinstance(evaluate_pipeline, str): module, funcname = evaluate_pipeline.rsplit(":", 1) @@ -476,6 +459,8 @@ def __call__( "evaluate_pipeline must be a callable or a string in the format" "'module:function'." ) + if isinstance(space, PipelineSpace): + _eval = adjust_evaluation_pipeline_for_neps_space(_eval, space) _launch_runtime( evaluation_fn=_eval, # type: ignore @@ -492,12 +477,12 @@ def __call__( overwrite_optimization_dir=overwrite_root_directory, sample_batch_size=sample_batch_size, worker_id=worker_id, + pipeline_space=pipeline_space, ) post_run_csv(root_directory) root_directory = Path(root_directory) summary_dir = root_directory / "summary" - trajectory_of_improvements(root_directory) logger.info( "The summary folder has been created, which contains csv and txt files with" "the output of all data in the run (short.csv - only the best; full.csv - " @@ -541,15 +526,18 @@ def save_pipeline_results( ) -def import_trials( - pipeline_space: SearchSpace, +def import_trials( # noqa: C901 evaluated_trials: Sequence[tuple[Mapping[str, Any], UserResultDict],], root_directory: Path | str, + pipeline_space: SearchSpace | dict | PipelineSpace | None = None, + overwrite_root_directory: bool = False, # noqa: FBT001, FBT002 optimizer: ( OptimizerChoice | Mapping[str, Any] | tuple[OptimizerChoice, Mapping[str, Any]] - | Callable[Concatenate[SearchSpace, ...], AskFunction] + | Callable[Concatenate[SearchSpace, ...], AskFunction] # Hack, while we transit + | Callable[Concatenate[PipelineSpace, ...], AskFunction] # from SearchSpace to + | Callable[Concatenate[SearchSpace | PipelineSpace, ...], AskFunction] # Pipeline | CustomOptimizer | Literal["auto"] ) = "auto", @@ -562,11 +550,16 @@ def import_trials( removes duplicates, and updates the optimization state accordingly. Args: - pipeline_space (SearchSpace): The search space used for the optimization. evaluated_trials (Sequence[tuple[Mapping[str, Any], UserResultDict]]): A sequence of tuples, each containing a configuration dictionary and its corresponding result. root_directory (Path or str): The root directory of the NePS run. + pipeline_space (SearchSpace | dict | PipelineSpace | None): The pipeline space + used for the optimization. If None, will attempt to load from the + root_directory. If provided and a pipeline space exists on disk, they + will be validated to match. + overwrite_root_directory (bool, optional): If True, overwrite the existing + root directory. Defaults to False. optimizer: The optimizer to use for importing trials. Can be a string, mapping, tuple, callable, or CustomOptimizer. Defaults to "auto". @@ -575,7 +568,8 @@ def import_trials( None Raises: - ValueError: If any configuration or result is invalid. + ValueError: If any configuration or result is invalid, or if pipeline_space + cannot be determined (neither provided nor found on disk). FileNotFoundError: If the root directory does not exist. Exception: For unexpected errors during trial import. @@ -587,12 +581,65 @@ def import_trials( ... ({"param1": 0.5, "param2": 10}, ... UserResultDict(objective_to_minimize=-5.0)), ... ] - >>> neps.import_trials(pipeline_space, evaluated_trials, "my_results") + >>> neps.import_trials(evaluated_trials, "my_results", pipeline_space) """ if isinstance(root_directory, str): root_directory = Path(root_directory) - optimizer_ask, optimizer_info = load_optimizer(optimizer, pipeline_space) + # Try to load pipeline_space from disk if not provided + if pipeline_space is None: + if root_directory.exists() and not overwrite_root_directory: + try: + pipeline_space = load_pipeline_space(root_directory) + logger.info( + "Loaded pipeline space from disk. Importing trials with " + f"existing pipeline space from {root_directory}" + ) + except (FileNotFoundError, ValueError) as e: + # If loading fails, we'll error below + logger.debug(f"Could not load pipeline space from disk: {e}") + + # If still None, raise error + if pipeline_space is None: + raise ValueError( + "pipeline_space is required when importing trials to a new run. " + "For importing to an existing run, the pipeline space will be loaded " + f"from disk. No existing pipeline space found at: {root_directory}" + ) + # Note: If pipeline_space is provided, it will be validated against the one on disk + # by NePSState.create_or_load() after necessary conversions are applied + + neps_classic_space_compatibility = check_neps_space_compatibility(optimizer) + if neps_classic_space_compatibility in ["both", "classic"] and isinstance( + pipeline_space, PipelineSpace + ): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space: + pipeline_space = converted_space + space = convert_to_space(pipeline_space) + + if neps_classic_space_compatibility == "neps" and not isinstance( + space, PipelineSpace + ): + space = convert_classic_to_neps_search_space(space) + + # Optimizer check, if the pipeline space is a Pipeline and the optimizer is not a NEPS + # algorithm, we raise an error, as the optimizer is not compatible. + if isinstance(space, PipelineSpace) and neps_classic_space_compatibility == "classic": + raise ValueError( + f"The provided optimizer {optimizer} is not compatible with this complex" + " pipeline space. Please use one that is, such as 'random_search'," + " 'hyperband', 'priorband', or 'complex_random_search'." + ) + + optimizer_ask, optimizer_info = load_optimizer(optimizer, space) + + if overwrite_root_directory and root_directory.exists(): + logger.info( + f"Overwriting root directory '{root_directory}' as" + " `overwrite_root_directory=True`." + ) + shutil.rmtree(root_directory) state = NePSState.create_or_load( root_directory, @@ -600,13 +647,14 @@ def import_trials( optimizer_state=OptimizationState( budget=None, seed_snapshot=SeedSnapshot.new_capture(), shared_state={} ), + pipeline_space=space, ) normalized_trials = [] for config, result in evaluated_trials: - _validate_imported_config(pipeline_space, config) + _validate_imported_config(space, config) _validate_imported_result(result) - normalized_config = _normalize_imported_config(pipeline_space, config) + normalized_config = _normalize_imported_config(space, config) normalized_trials.append((normalized_config, result)) with state._trial_lock.lock(): @@ -615,11 +663,17 @@ def import_trials( existing_configs = [ tuple(sorted(t.config.items())) for t in state_trials.values() ] + num_before_dedup = len(normalized_trials) normalized_trials = [ t for t in normalized_trials if tuple(sorted(t[0].items())) not in existing_configs ] + num_duplicates = num_before_dedup - len(normalized_trials) + if num_duplicates > 0: + logger.info( + f"Skipped {num_duplicates} duplicate trial(s) (already exist in state)." + ) imported_trials = optimizer_ask.import_trials( external_evaluations=normalized_trials, @@ -629,4 +683,396 @@ def import_trials( state.lock_and_import_trials(imported_trials, worker_id="external") -__all__ = ["import_trials", "run", "save_pipeline_results"] +def create_config( # noqa: C901 + pipeline_space: PipelineSpace | None = None, + root_directory: Path | str | None = None, +) -> tuple[Mapping[str, Any], dict[str, Any]]: + """Create a configuration by prompting the user for input. + + Args: + pipeline_space: The pipeline space to create a configuration for. + If None, will attempt to load from + `root_directory/pipeline_space.pkl` if `root_directory` is + provided. + root_directory: The root directory to load the pipeline space from + if `pipeline_space` is None. + + Returns: + A tuple containing the created configuration dictionary and the + sampled pipeline. + """ + from neps.space.neps_spaces.neps_space import NepsCompatConverter + from neps.space.neps_spaces.sampling import IOSampler + + # Try to load pipeline_space from disk if path is provided + if root_directory: + try: + loaded_space = load_pipeline_space(root_directory) + except (FileNotFoundError, ValueError) as e: + # If loading fails, we'll error below + raise ValueError( + f"Could not load pipeline space from disk at {root_directory}: {e}" + ) from e + # Validate loaded space is a PipelineSpace + if not isinstance(loaded_space, PipelineSpace): + raise ValueError( + "create_config only supports PipelineSpace. The loaded space " + f"from {root_directory} is not a PipelineSpace." + ) + + if pipeline_space is None: + pipeline_space = loaded_space + else: + # Validate provided pipeline_space is a PipelineSpace + if not isinstance(pipeline_space, PipelineSpace): + raise ValueError( + "create_config only supports PipelineSpace. The provided " + "pipeline_space is not a PipelineSpace." + ) + + # Validate provided pipeline_space matches loaded one + import pickle + + if pickle.dumps(loaded_space) != pickle.dumps(pipeline_space): + raise ValueError( + "The pipeline_space provided does not match the one saved on" + " disk.\nPipeline space location:" + f" {Path(root_directory) / 'pipeline_space.pkl'}\nPlease either:\n" + " 1. Don't provide pipeline_space (it will be loaded automatically)," + " or\n 2. Provide the same pipeline_space that was used in" + " neps.run()" + ) + elif pipeline_space is None: + raise ValueError( + "pipeline_space or root_directory is required when creating a configuration." + ) + + resolved_pipeline, resolution_context = resolve( + pipeline_space, domain_sampler=IOSampler() + ) + + # Print the resolved pipeline + + pipeline_dict = dict(**resolved_pipeline.get_attrs()) + + for name, value in pipeline_dict.items(): + if isinstance(value, Operation): + # If the operator is a not a string, we convert it to a callable. + if isinstance(value.operator, str): + pipeline_dict[name] = format_value(value) + else: + pipeline_dict[name] = convert_operation_to_callable(value) + + return NepsCompatConverter.to_neps_config(resolution_context), pipeline_dict + + +def load_config( # noqa: C901, PLR0912, PLR0915 + pipeline_space: PipelineSpace | SearchSpace | None = None, + config: dict[str, Any] | None = None, + config_path: Path | str | None = None, + config_id: str | None = None, +) -> dict[str, Any]: + """Load a configuration from a neps config file. + + Args: + pipeline_space: The pipeline space used to generate the configuration. + If None, will attempt to load from the NePSState directory. + config: Optional configuration dictionary to return directly. + config_path: Path to the neps config file. + config_id: Optional config id to load, when only giving results folder. + + Returns: + The loaded configuration as a dictionary. + + Raises: + ValueError: If pipeline_space is not provided and cannot be loaded from disk. + """ + from neps.space.neps_spaces.neps_space import NepsCompatConverter + from neps.space.neps_spaces.sampling import OnlyPredefinedValuesSampler + + if config is None: + if config_path is None: + raise ValueError("Either config or config_path must be provided.") + # Try to load pipeline_space from NePSState if not provided + state = None # Track state for later use in config loading + + if pipeline_space is None: + try: + # Extract the root directory from config_path + str_path_temp = str(config_path) + if "/configs/" in str_path_temp or "\\configs\\" in str_path_temp: + root_dir = Path( + str_path_temp.split("/configs/")[0].split("\\configs\\")[0] + ) + # If no /configs/ in path, assume it's either: + # 1. The root directory itself + # 2. A direct config file path (ends with .yaml/.yml) + elif str_path_temp.endswith((".yaml", ".yml")): + # It's a direct config file path, go up two levels + root_dir = Path(str_path_temp).parent.parent + else: + # It's the root directory itself + root_dir = Path(str_path_temp) + + state = NePSState.create_or_load(path=root_dir, load_only=True) + pipeline_space = state.lock_and_get_search_space() + + if pipeline_space is None: + raise ValueError( + "Could not load pipeline_space from disk. " + "Please provide pipeline_space argument or ensure " + "the NePSState was created with search_space saved." + ) + except Exception as e: + raise ValueError( + f"pipeline_space not provided and could not be loaded from disk: {e}" + ) from e + else: + # User provided a pipeline_space - validate it matches the one on disk + from neps.exceptions import NePSError + + try: + str_path_temp = str(config_path) + if "/configs/" in str_path_temp or "\\configs\\" in str_path_temp: + root_dir = Path( + str_path_temp.split("/configs/")[0].split("\\configs\\")[0] + ) + # If no /configs/ in path, assume it's either: + # 1. The root directory itself + # 2. A direct config file path (ends with .yaml/.yml) + elif str_path_temp.endswith((".yaml", ".yml")): + # It's a direct config file path, go up two levels + root_dir = Path(str_path_temp).parent.parent + else: + # It's the root directory itself + root_dir = Path(str_path_temp) + + state = NePSState.create_or_load(path=root_dir, load_only=True) + disk_space = state.lock_and_get_search_space() + + if disk_space is not None: + # Validate that provided space matches disk space + import pickle + + if pickle.dumps(disk_space) != pickle.dumps(pipeline_space): + raise NePSError( + "The pipeline_space provided does not match the one saved on" + " disk.\\nPipeline space location:" + f" {root_dir / 'pipeline_space.pkl'}\\nPlease either:\\n 1." + " Don't provide pipeline_space (it will be loaded" + " automatically), or\\n 2. Provide the same pipeline_space" + " that was used in neps.run()" + ) + except NePSError: + raise + except Exception: # noqa: S110, BLE001 + # If we can't load/validate, just continue with provided space + pass + + # Determine config_id from path + str_path = str(config_path) + trial_id = None + + if not str_path.endswith(".yaml") and not str_path.endswith(".yml"): + if str_path.removesuffix("/").split("/")[-1].startswith("config_"): + # Extract trial_id from path like "configs/config_1" + # or "configs/config_1_rung_0" + trial_id = str_path.removesuffix("/").split("/")[-1] + else: + if config_id is None: + raise ValueError( + "When providing a results folder, you must also provide a" + " config_id." + ) + trial_id = config_id + else: + # Extract trial_id from yaml path like "configs/config_1/config.yaml" + path_parts = str_path.replace("\\", "/").split("/") + for i, part in enumerate(path_parts): + if part == "configs" and i + 1 < len(path_parts): + trial_id = path_parts[i + 1] + break + + # Use the locked method from NePSState to safely read the trial + if trial_id is not None and state is not None: + try: + trial = state.lock_and_get_trial_by_id(trial_id) + config_dict = dict(trial.config) # Convert Mapping to dict + except Exception: # noqa: BLE001 + # Fallback to direct file read if trial can't be loaded + str_path_fallback = str(config_path) + if not str_path_fallback.endswith( + ".yaml" + ) and not str_path_fallback.endswith(".yml"): + str_path_fallback += "/config.yaml" + config_path = Path(str_path_fallback) + with config_path.open("r") as f: + config_dict = yaml.load(f, Loader=yaml.SafeLoader) + else: + # Fallback to direct file read + str_path_fallback = str(config_path) + if not str_path_fallback.endswith(".yaml") and not str_path_fallback.endswith( + ".yml" + ): + str_path_fallback += "/config.yaml" + config_path = Path(str_path_fallback) + with config_path.open("r") as f: + config_dict = yaml.load(f, Loader=yaml.SafeLoader) + + else: + config_dict = config + + if ( + any(NepsCompatConverter._SAMPLING_PREFIX in key for key in config_dict) + and pipeline_space is None + ): + raise ValueError( + "The provided NePS-space config requires the correct pipeline_space to be " + "resolved. Please provide a pipeline_space argument to load_config." + ) + + # Handle different pipeline space types + if not isinstance(pipeline_space, PipelineSpace): + # For SearchSpace (classic), just return the config dict + return dict(config_dict) if isinstance(config_dict, Mapping) else config_dict + + # For PipelineSpace, resolve it + converted_dict = NepsCompatConverter.from_neps_config(config_dict) + + pipeline, _ = resolve( + pipeline_space, + domain_sampler=OnlyPredefinedValuesSampler(converted_dict.predefined_samplings), + environment_values=converted_dict.environment_values, + ) + + # Print the resolved pipeline + + pipeline_dict = dict(**pipeline.get_attrs()) + + for name, value in pipeline_dict.items(): + if isinstance(value, Operation): + # If the operator is a not a string, we convert it to a callable. + if isinstance(value.operator, str): + pipeline_dict[name] = format_value(value) + else: + pipeline_dict[name] = convert_operation_to_callable(value) + + return pipeline_dict + + +def load_pipeline_space( + root_directory: str | Path, +) -> PipelineSpace | SearchSpace: + """Load the pipeline space from a neps run directory. + + This is a convenience function that loads the pipeline space that was saved + during a neps.run() call. The pipeline space is automatically saved to disk + and can be loaded to inspect it or use it with other neps utilities. + + Args: + root_directory: Path to the neps results directory (the same path + that was passed to neps.run()). + + Returns: + The pipeline space that was used in the neps run. + + Raises: + FileNotFoundError: If no neps state is found at the given path. + ValueError: If no pipeline space was saved in the neps run. + + Example: + ```python + # After running neps + neps.run( + evaluate_pipeline=my_function, + pipeline_space=MySpace(), + root_directory="results", + ) + + # Later, load the space + space = neps.load_pipeline_space("results") + ``` + """ + from neps.state import NePSState + + root_directory = Path(root_directory) + + try: + state = NePSState.create_or_load(path=root_directory, load_only=True) + pipeline_space = state.lock_and_get_search_space() + + if pipeline_space is None: + raise ValueError( + f"No pipeline space was saved in the neps run at: {root_directory}\n" + "This can happen if the run was created before pipeline space " + "persistence was added, or if the pipeline_space.pkl file was deleted." + ) + + return pipeline_space + except FileNotFoundError as e: + raise FileNotFoundError( + f"No neps state found at: {root_directory}\n" + "Please provide a valid neps results directory." + ) from e + + +def load_optimizer_info( + root_directory: str | Path, +) -> OptimizerInfo: + """Load the optimizer information from a neps run directory. + + This function loads the optimizer metadata that was saved during a neps.run() + call, including the optimizer name and its configuration parameters. This is + useful for inspecting what optimizer was used and with what settings. + + Args: + root_directory: Path to the neps results directory (the same path + that was passed to neps.run()). + + Returns: + A dictionary containing: + - 'name': The name of the optimizer (e.g., 'bayesian_optimization') + - 'info': Additional optimizer configuration (e.g., initialization kwargs) + + Raises: + FileNotFoundError: If no neps state is found at the given path. + + Example: + ```python + # After running neps + neps.run( + evaluate_pipeline=my_function, + pipeline_space=MySpace(), + root_directory="results", + optimizer="bayesian_optimization", + ) + + # Later, check what optimizer was used + optimizer_info = neps.load_optimizer_info("results") + print(f"Optimizer: {optimizer_info['name']}") + print(f"Config: {optimizer_info['info']}") + ``` + """ + from neps.state import NePSState + + root_directory = Path(root_directory) + + try: + state = NePSState.create_or_load(path=root_directory, load_only=True) + return state.lock_and_get_optimizer_info() + except FileNotFoundError as e: + raise FileNotFoundError( + f"No neps state found at: {root_directory}\n" + "Please provide a valid neps results directory." + ) from e + + +__all__ = [ + "create_config", + "import_trials", + "load_config", + "load_optimizer_info", + "load_pipeline_space", + "run", + "save_pipeline_results", +] diff --git a/neps/normalization.py b/neps/normalization.py index 1e07ba9e5..a16a3743a 100644 --- a/neps/normalization.py +++ b/neps/normalization.py @@ -6,13 +6,17 @@ from collections.abc import Mapping from typing import TYPE_CHECKING +from neps.space import SearchSpace + if TYPE_CHECKING: - from neps.space import SearchSpace + from neps.space.neps_spaces.parameters import PipelineSpace logger = logging.getLogger(__name__) -def _normalize_imported_config(space: SearchSpace, config: Mapping[str, float]) -> dict: +def _normalize_imported_config( + space: SearchSpace | PipelineSpace, config: Mapping[str, float] +) -> dict: """Completes a configuration by adding default values for missing fidelities. Args: @@ -22,16 +26,44 @@ def _normalize_imported_config(space: SearchSpace, config: Mapping[str, float]) Returns: A new, completed configuration dictionary. """ - all_param_keys = set(space.searchables.keys()) | set(space.fidelities.keys()) + if isinstance(space, SearchSpace): + all_param_keys = set(space.searchables.keys()) | set(space.fidelities.keys()) + # copy to avoid modifying the original config + normalized_conf = dict(config) + for key, param in space.fidelities.items(): + if key not in normalized_conf: + normalized_conf[key] = param.upper + extra_keys = set(normalized_conf.keys()) - all_param_keys + else: + # For PipelineSpace, we need to generate the prefixed keys + # Import here to avoid circular import + from neps.space.neps_spaces.neps_space import ( + NepsCompatConverter, + ) + + # copy to avoid modifying the original config + normalized_conf = dict(config) - # copy to avoid modifying the original config - normalized_conf = dict(config) + for key, fid_param in space.fidelity_attrs.items(): + fid_key = NepsCompatConverter._ENVIRONMENT_PREFIX + key + if fid_key not in normalized_conf: + normalized_conf[fid_key] = fid_param.upper + # For PipelineSpace, filter out keys that match the expected patterns + # Import here to avoid circular import (needed for prefix constants) + from neps.space.neps_spaces.neps_space import NepsCompatConverter - for key, param in space.fidelities.items(): - if key not in normalized_conf: - normalized_conf[key] = param.upper + extra_keys = set() + for key in normalized_conf: + if not key.startswith( + ( + NepsCompatConverter._SAMPLING_PREFIX, + NepsCompatConverter._ENVIRONMENT_PREFIX, + ) + ): + # It has no prefix. + # TODO: It might still be unnecessary, but it will not hurt. + extra_keys.add(key) - extra_keys = set(normalized_conf.keys()) - all_param_keys if extra_keys: logger.warning(f"Unknown parameters in config: {extra_keys}, discarding them") for k in extra_keys: diff --git a/neps/optimizers/__init__.py b/neps/optimizers/__init__.py index 9b97790a9..29b07baad 100644 --- a/neps/optimizers/__init__.py +++ b/neps/optimizers/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping +from functools import partial from typing import TYPE_CHECKING, Any, Concatenate, Literal from neps.optimizers.algorithms import ( @@ -14,11 +15,12 @@ if TYPE_CHECKING: from neps.space import SearchSpace + from neps.space.neps_spaces.parameters import PipelineSpace def _load_optimizer_from_string( optimizer: OptimizerChoice | Literal["auto"], - space: SearchSpace, + space: SearchSpace | PipelineSpace, *, optimizer_kwargs: Mapping[str, Any] | None = None, ) -> tuple[AskFunction, OptimizerInfo]: @@ -37,7 +39,10 @@ def _load_optimizer_from_string( keywords = extract_keyword_defaults(optimizer_build) optimizer_kwargs = optimizer_kwargs or {} - opt = optimizer_build(space, **optimizer_kwargs) + optimizer_kwargs = dict(optimizer_kwargs) # Make mutable copy + if _optimizer == "primo": + optimizer_kwargs["prior_centers"] = optimizer_kwargs.get("prior_centers", {}) + opt = optimizer_build(space, **optimizer_kwargs) # type: ignore info = OptimizerInfo(name=_optimizer, info={**keywords, **optimizer_kwargs}) return opt, info @@ -47,11 +52,13 @@ def load_optimizer( OptimizerChoice | Mapping[str, Any] | tuple[OptimizerChoice, Mapping[str, Any]] - | Callable[Concatenate[SearchSpace, ...], AskFunction] + | Callable[Concatenate[SearchSpace, ...], AskFunction] # Hack, while we transit + | Callable[Concatenate[PipelineSpace, ...], AskFunction] # from SearchSpace to + | Callable[Concatenate[SearchSpace | PipelineSpace, ...], AskFunction] # Pipeline | CustomOptimizer | Literal["auto"] ), - space: SearchSpace, + space: SearchSpace | PipelineSpace, ) -> tuple[AskFunction, OptimizerInfo]: match optimizer: # Predefined string (including "auto") @@ -68,9 +75,27 @@ def load_optimizer( # Provided optimizer initializer case _ if callable(optimizer): + inner_optimizer = None + if isinstance(optimizer, partial): + inner_optimizer = optimizer.func + while isinstance(inner_optimizer, partial): + inner_optimizer = inner_optimizer.func + else: + inner_optimizer = optimizer keywords = extract_keyword_defaults(optimizer) - _optimizer = optimizer(space) - info = OptimizerInfo(name=optimizer.__name__, info=keywords) + + # Error catch and type ignore needed while we transition from SearchSpace to + # Pipeline + try: + _optimizer = inner_optimizer(space, **keywords) # type: ignore + except TypeError as e: + raise TypeError( + f"Optimizer {inner_optimizer} does not accept a space of type" + f" {type(space)}." + ) from e + + info = OptimizerInfo(name=inner_optimizer.__name__, info=keywords) + return _optimizer, info # Custom optimizer, we create it diff --git a/neps/optimizers/algorithms.py b/neps/optimizers/algorithms.py index e80377e1e..fa14cfee6 100644 --- a/neps/optimizers/algorithms.py +++ b/neps/optimizers/algorithms.py @@ -1,4 +1,6 @@ -"""The selection of optimization algorithms available in NePS. +"""NePS Algorithms +=========== +The selection of optimization algorithms available in NePS. This module conveniently starts with 'a' to be at the top and is where most of the code documentation for optimizers can be found. @@ -32,12 +34,32 @@ from neps.optimizers.ifbo import IFBO from neps.optimizers.models.ftpfn import FTPFNSurrogate from neps.optimizers.mopriors import MOPriorSampler +from neps.optimizers.neps_bracket_optimizer import _NePSBracketOptimizer +from neps.optimizers.neps_local_and_incumbent import NePSLocalPriorIncumbentSampler +from neps.optimizers.neps_priorband import NePSPriorBandSampler +from neps.optimizers.neps_random_search import ( + NePSComplexRandomSearch, + NePSRandomSearch, +) +from neps.optimizers.neps_regularized_evolution import NePSRegularizedEvolution from neps.optimizers.optimizer import AskFunction # noqa: TC001 from neps.optimizers.primo import PriMO from neps.optimizers.priorband import PriorBandSampler from neps.optimizers.random_search import RandomSearch from neps.sampling import Prior, Sampler, Uniform from neps.space.encoding import CategoricalToUnitNorm, ConfigEncoder +from neps.space.neps_spaces.neps_space import ( + NepsCompatConverter, + convert_neps_to_classic_search_space, +) +from neps.space.neps_spaces.parameters import ( + PipelineSpace, +) +from neps.space.neps_spaces.sampling import ( + DomainSampler, + PriorOrFallbackSampler, + RandomSampler, +) from neps.space.parsing import convert_mapping if TYPE_CHECKING: @@ -46,11 +68,12 @@ from neps.optimizers.utils.brackets import Bracket from neps.space import SearchSpace + logger = logging.getLogger(__name__) -def _bo( - pipeline_space: SearchSpace, +def _bo( # noqa: C901, PLR0912 + pipeline_space: SearchSpace | PipelineSpace, *, initial_design_size: int | Literal["ndim"] = "ndim", use_priors: bool, @@ -78,7 +101,7 @@ def _bo( If using `cost`, cost must be provided in the reports of the trials. sample_prior_first: Whether to sample the default configuration first. - ignore_fidelity: Whether to ignore fidelity when sampling. + ignore_fidelity: Whether to ignore_fidelity when sampling. In this case, the max fidelity is always used. device: Device to use for the optimization. reference_point: The reference point to use for multi-objective optimization. @@ -87,6 +110,15 @@ def _bo( ValueError: if initial_design_size < 1 ValueError: if fidelity is not None and ignore_fidelity is False """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) if not ignore_fidelity and pipeline_space.fidelity is not None: raise ValueError( "Fidelities are not supported for BayesianOptimization. Consider setting the" @@ -135,14 +167,16 @@ def _bo( def _bracket_optimizer( # noqa: C901, PLR0912, PLR0915 - pipeline_space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, bracket_type: Literal["successive_halving", "hyperband", "asha", "async_hb"], eta: int, - sampler: Literal["uniform", "prior", "priorband", "mopriorsampler"] - | PriorBandSampler - | MOPriorSampler - | Sampler, + sampler: ( + Literal["uniform", "prior", "priorband", "mopriorsampler"] + | PriorBandSampler + | MOPriorSampler + | Sampler + ), bayesian_optimization_kick_in_point: int | float | None, sample_prior_first: bool | Literal["highest_fidelity"], # NOTE: This is the only argument to get a default, since it @@ -212,6 +246,15 @@ def _bracket_optimizer( # noqa: C901, PLR0912, PLR0915 multi_objective: Whether to use multi-objective promotion strategies. Only used in case of multi-objective multi-fidelity algorithms. """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) if pipeline_space.fidelity is not None: fidelity_name, fidelity = pipeline_space.fidelity else: @@ -389,7 +432,16 @@ def _bracket_optimizer( # noqa: C901, PLR0912, PLR0915 ) -def determine_optimizer_automatically(space: SearchSpace) -> str: +def determine_optimizer_automatically( # noqa: PLR0911 + space: SearchSpace | PipelineSpace, +) -> str: + if isinstance(space, PipelineSpace): + has_prior = space.has_priors() + if space.fidelity_attrs and has_prior: + return "neps_priorband" + if space.fidelity_attrs and not has_prior: + return "neps_hyperband" + return "complex_random_search" has_prior = any( parameter.prior is not None for parameter in space.searchables.values() ) @@ -409,11 +461,11 @@ def determine_optimizer_automatically(space: SearchSpace) -> str: def random_search( - pipeline_space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, use_priors: bool = False, - ignore_fidelity: bool | Literal["highest fidelity"] = False, -) -> RandomSearch: + ignore_fidelity: bool | Literal["highest_fidelity"] = False, +) -> RandomSearch | NePSRandomSearch: """A simple random search algorithm that samples configurations uniformly at random. You may also `use_priors=` to sample from a distribution centered around your defined @@ -422,22 +474,32 @@ def random_search( Args: pipeline_space: The search space to sample from. use_priors: Whether to use priors when sampling. - ignore_fidelity: Whether to ignore fidelity when sampling. - In this case, the max fidelity is always used. + ignore_fidelity: Whether to ignore_fidelity when sampling. + Setting this to "highest_fidelity" will always sample at max fidelity. + Setting this to True will randomly sample from the fidelity like any other + parameter. """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + return neps_random_search( + pipeline_space, use_priors=use_priors, ignore_fidelity=ignore_fidelity + ) assert ignore_fidelity in ( True, False, - "highest fidelity", - ), "ignore_fidelity should be either True, False or 'highest fidelity'" + "highest_fidelity", + ), "ignore_fidelity should be either True, False or 'highest_fidelity'" if not ignore_fidelity and pipeline_space.fidelity is not None: raise ValueError( "Fidelities are not supported for RandomSearch. Consider setting the" " fidelity to a constant value, or setting ignore_fidelity to True to sample" - " from it like any other parameter or 'highest fidelity' to always sample at" + " from it like any other parameter or 'highest_fidelity' to always sample at" f" max fidelity. Got fidelity: {pipeline_space.fidelities} " ) - if ignore_fidelity in (True, "highest fidelity") and pipeline_space.fidelity is None: + if ignore_fidelity in (True, "highest_fidelity") and pipeline_space.fidelity is None: logger.warning( "Warning: You are using ignore_fidelity, but no fidelity is defined in the" " search space. Consider setting ignore_fidelity to False." @@ -447,7 +509,7 @@ def random_search( parameters = {**pipeline_space.searchables, **pipeline_space.fidelities} case False: parameters = {**pipeline_space.searchables} - case "highest fidelity": + case "highest_fidelity": parameters = {**pipeline_space.searchables} if use_priors and not any( @@ -480,36 +542,113 @@ def random_search( def grid_search( - pipeline_space: SearchSpace, - ignore_fidelity: bool = False, # noqa: FBT001, FBT002 + pipeline_space: SearchSpace | PipelineSpace, + *, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, + size_per_numerical_dimension: int = 5, ) -> GridSearch: """A simple grid search algorithm which discretizes the search space and evaluates all possible configurations. Args: pipeline_space: The search space to sample from. - ignore_fidelity: Whether to ignore fidelity when sampling. - In this case, the max fidelity is always used. + ignore_fidelity: Whether to ignore_fidelity when sampling. + Setting this to "highest_fidelity" will always sample at max fidelity. + Setting this to True will make a grid over the fidelity like any other + parameter. + size_per_numerical_dimension: The number of points to use per numerical + dimension when discretizing the space. """ from neps.optimizers.utils.grid import make_grid + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + return neps_grid_search( + pipeline_space, + ignore_fidelity=ignore_fidelity, + size_per_numerical_dimension=size_per_numerical_dimension, + ) + if any( parameter.prior is not None for parameter in pipeline_space.searchables.values() ): - raise ValueError("Grid search does not support priors.") + logger.warning("Grid search does not support priors, they will be ignored.") if ignore_fidelity and pipeline_space.fidelity is None: logger.warning( "Warning: You are using ignore_fidelity, but no fidelity is defined in the" " search space. Consider setting ignore_fidelity to False." ) + if not ignore_fidelity and pipeline_space.fidelity is not None: + raise ValueError( + "Fidelities are not supported for GridSearch natively. Consider setting the" + " fidelity to a constant value, or setting ignore_fidelity to True to sample" + " from it like any other parameter or 'highest_fidelity' to always sample at" + f" max fidelity. Got fidelity: {pipeline_space.fidelities} " + ) return GridSearch( - configs_list=make_grid(pipeline_space, ignore_fidelity=ignore_fidelity) + configs_list=make_grid( + pipeline_space, + ignore_fidelity=ignore_fidelity, + size_per_numerical_hp=size_per_numerical_dimension, + ) + ) + + +def neps_grid_search( + pipeline_space: PipelineSpace, + *, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, + size_per_numerical_dimension: int = 5, +) -> GridSearch: + """A simple grid search algorithm which discretizes the search + space and evaluates all possible configurations. + + Args: + pipeline_space: The search space to sample from. + ignore_fidelity: Whether to ignore_fidelity when sampling. + Setting this to "highest_fidelity" will always sample at max fidelity. + Setting this to True will make a grid over the fidelity like any other + parameter. + size_per_numerical_dimension: The number of points to use per numerical + dimension when discretizing the space. + """ + from neps.optimizers.utils.grid import make_grid + + if not isinstance(pipeline_space, PipelineSpace): + raise ValueError( + "This optimizer only supports NePS spaces, please use a classic" + " search space-compatible optimizer." + ) + if pipeline_space.has_priors(): + logger.warning("Grid search does not support priors, they will be ignored.") + if not pipeline_space.fidelity_attrs and ignore_fidelity: + logger.warning( + "Warning: You are using ignore_fidelity, but no fidelity is defined in the" + " search space. Consider setting ignore_fidelity to False." + ) + if pipeline_space.fidelity_attrs and not ignore_fidelity: + raise ValueError( + "Fidelities are not supported for GridSearch natively. Consider setting the" + " fidelity to a constant value, or setting ignore_fidelity to True to sample" + " from it like any other parameter or 'highest_fidelity' to always sample at" + f" max fidelity. Got fidelity: {pipeline_space.fidelity_attrs} " + ) + + return GridSearch( + configs_list=make_grid( + pipeline_space, + ignore_fidelity=ignore_fidelity, + size_per_numerical_hp=size_per_numerical_dimension, + ) ) def ifbo( - pipeline_space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, step_size: int | float = 1, use_priors: bool = False, @@ -559,6 +698,15 @@ def ifbo( surrogate_path: Path to the surrogate model to use surrogate_version: Version of the surrogate model to use """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) from neps.optimizers.ifbo import _adjust_space_to_match_stepsize if pipeline_space.fidelity is None: @@ -628,7 +776,7 @@ def ifbo( def successive_halving( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, sampler: Literal["uniform", "prior"] = "uniform", eta: int = 3, @@ -681,7 +829,7 @@ def successive_halving( or `#!python sampler="prior"`. Args: - space: The search space to sample from. + pipeline_space: The search space to sample from. eta: The reduction factor used for building brackets early_stopping_rate: Determines the number of rungs in a bracket Choosing 0 creates maximal rungs given the fidelity bounds. @@ -694,10 +842,19 @@ def successive_halving( values in the search space. sample_prior_first: Whether to sample the prior configuration first, - and if so, should it be at the highest fidelity level. + and if so, should it be at the highest_fidelity level. """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) return _bracket_optimizer( - pipeline_space=space, + pipeline_space=pipeline_space, bracket_type="successive_halving", eta=eta, early_stopping_rate=early_stopping_rate, @@ -710,12 +867,12 @@ def successive_halving( def hyperband( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, eta: int = 3, sampler: Literal["uniform", "prior"] = "uniform", sample_prior_first: bool | Literal["highest_fidelity"] = False, -) -> BracketOptimizer: +) -> BracketOptimizer | _NePSBracketOptimizer: """Another bandit-based optimization algorithm that uses a _fidelity_ parameter, very similar to [`successive_halving`][neps.optimizers.algorithms.successive_halving], but hedges a bit more on the safe side, just incase your _fidelity_ parameters @@ -747,7 +904,7 @@ def hyperband( as this algorithm could be considered an extension of it. Args: - space: The search space to sample from. + pipeline_space: The search space to sample from. eta: The reduction factor used for building brackets sampler: The type of sampling procedure to use: @@ -758,10 +915,21 @@ def hyperband( values in the search space. sample_prior_first: Whether to sample the prior configuration first, - and if so, should it be at the highest fidelity level. + and if so, should it be at the highest_fidelity level. """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space: + pipeline_space = converted_space + else: + return neps_hyperband( + pipeline_space, + eta=eta, + sampler=sampler, + sample_prior_first=sample_prior_first, + ) return _bracket_optimizer( - pipeline_space=space, + pipeline_space=pipeline_space, bracket_type="hyperband", eta=eta, sampler=sampler, @@ -773,8 +941,42 @@ def hyperband( ) +def neps_hyperband( + pipeline_space: PipelineSpace, + *, + eta: int = 3, + sampler: Literal["uniform", "prior"] = "uniform", + sample_prior_first: bool | Literal["highest_fidelity"] = False, +) -> _NePSBracketOptimizer: + """ + Hyperband optimizer for NePS search spaces. + Args: + pipeline_space: The search space to sample from. + eta: The reduction factor used for building brackets + sampler: The type of sampling procedure to use: + + * If `#!python "uniform"`, samples uniformly from the space when + it needs to sample. + * If `#!python "prior"`, samples from the prior + distribution built from the `prior` and `prior_confidence` + values in the search space. + + sample_prior_first: Whether to sample the prior configuration first, + and if so, should it be at the highest_fidelity level. + """ + return _neps_bracket_optimizer( + pipeline_space=pipeline_space, + bracket_type="hyperband", + eta=eta, + sampler="prior" if sampler == "prior" else "uniform", + sample_prior_first=sample_prior_first, + early_stopping_rate=None, + sampler_kwargs={}, + ) + + def mo_hyperband( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, eta: int = 3, sampler: Literal["uniform", "prior"] = "uniform", @@ -784,8 +986,17 @@ def mo_hyperband( """Multi-objective version of hyperband using the same candidate selection method as MOASHA. """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) return _bracket_optimizer( - pipeline_space=space, + pipeline_space=pipeline_space, bracket_type="hyperband", eta=eta, sampler=sampler, @@ -800,7 +1011,7 @@ def mo_hyperband( def asha( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, eta: int = 3, early_stopping_rate: int = 0, @@ -836,7 +1047,7 @@ def asha( as this algorithm could be considered an extension of it. Args: - space: The search space to sample from. + pipeline_space: The search space to sample from. eta: The reduction factor used for building brackets sampler: The type of sampling procedure to use: @@ -847,11 +1058,19 @@ def asha( values in the search space. sample_prior_first: Whether to sample the prior configuration first, - and if so, should it be at the highest fidelity. + and if so, should it be at the highest_fidelity. """ - + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) return _bracket_optimizer( - pipeline_space=space, + pipeline_space=pipeline_space, bracket_type="asha", eta=eta, early_stopping_rate=early_stopping_rate, @@ -864,7 +1083,7 @@ def asha( def moasha( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, eta: int = 3, early_stopping_rate: int = 0, @@ -872,8 +1091,17 @@ def moasha( sample_prior_first: bool | Literal["highest_fidelity"] = False, mo_selector: Literal["nsga2", "epsnet"] = "epsnet", ) -> BracketOptimizer: + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) return _bracket_optimizer( - pipeline_space=space, + pipeline_space=pipeline_space, bracket_type="asha", eta=eta, early_stopping_rate=early_stopping_rate, @@ -888,7 +1116,7 @@ def moasha( def async_hb( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, eta: int = 3, sampler: Literal["uniform", "prior"] = "uniform", @@ -922,7 +1150,7 @@ def async_hb( takes elements from each. Args: - space: The search space to sample from. + pipeline_space: The search space to sample from. eta: The reduction factor used for building brackets sampler: The type of sampling procedure to use: @@ -934,8 +1162,17 @@ def async_hb( sample_prior_first: Whether to sample the prior configuration first. """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) return _bracket_optimizer( - pipeline_space=space, + pipeline_space=pipeline_space, bracket_type="async_hb", eta=eta, sampler=sampler, @@ -948,13 +1185,13 @@ def async_hb( def priorband( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, eta: int = 3, sample_prior_first: bool | Literal["highest_fidelity"] = False, base: Literal["successive_halving", "hyperband", "asha", "async_hb"] = "hyperband", bayesian_optimization_kick_in_point: int | float | None = None, -) -> BracketOptimizer: +) -> BracketOptimizer | _NePSBracketOptimizer: """Priorband is also a bandit-based optimization algorithm that uses a _fidelity_, providing a general purpose sampling extension to other algorithms. It makes better use of the prior information you provide in the search space along with the fact @@ -984,7 +1221,7 @@ def priorband( See: https://openreview.net/forum?id=uoiwugtpCH¬eId=xECpK2WH6k Args: - space: The search space to sample from. + pipeline_space: The search space to sample from. eta: The reduction factor used for building brackets sample_prior_first: Whether to sample the prior configuration first. base: The base algorithm to use for the bracketing. @@ -992,13 +1229,29 @@ def priorband( `N` * `maximum_fidelity` worth of fidelity has been evaluated, proceed with bayesian optimization when sampling a new configuration. """ - if all(parameter.prior is None for parameter in space.searchables.values()): + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + if bayesian_optimization_kick_in_point is not None: + raise ValueError( + "The priorband variant for this complex search space does not" + " support a bayesian optimization kick-in point yet." + ) + return neps_priorband( + pipeline_space, + eta=eta, + sample_prior_first=sample_prior_first, + base=base, + ) + if all(parameter.prior is None for parameter in pipeline_space.searchables.values()): logger.warning( "Warning: No priors are defined in the search space, priorband will sample" " uniformly. Consider using hyperband instead." ) return _bracket_optimizer( - pipeline_space=space, + pipeline_space=pipeline_space, bracket_type=base, eta=eta, sampler="priorband", @@ -1010,7 +1263,7 @@ def priorband( def bayesian_optimization( - space: SearchSpace, + pipeline_space: SearchSpace, *, initial_design_size: int | Literal["ndim"] = "ndim", cost_aware: bool | Literal["log"] = False, @@ -1047,7 +1300,7 @@ def bayesian_optimization( acquisition function. Args: - space: The search space to sample from. + pipeline_space: The search space to sample from. initial_design_size: Number of samples used before using the surrogate model. If "ndim", it will use the number of parameters in the search space. cost_aware: Whether to consider reported "cost" from configurations in decision @@ -1068,23 +1321,34 @@ def bayesian_optimization( optimization. If `None`, the reference point will be calculated automatically. """ + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) - if not ignore_fidelity and space.fidelity is not None: + if not ignore_fidelity and pipeline_space.fidelity is not None: raise ValueError( "Fidelities are not supported for BayesianOptimization. Consider setting the" " fidelity to a constant value or ignoring it using ignore_fidelity to" - f" always sample at max fidelity. Got fidelity: {space.fidelities} " + f" always sample at max fidelity. Got fidelity: {pipeline_space.fidelities} " ) - if ignore_fidelity and space.fidelity is None: + if ignore_fidelity and pipeline_space.fidelity is None: logger.warning( "Warning: You are using ignore_fidelity, but no fidelity is defined in the" " search space. Consider setting ignore_fidelity to False." ) - if any(parameter.prior is not None for parameter in space.searchables.values()): + if any( + parameter.prior is not None for parameter in pipeline_space.searchables.values() + ): priors = [ parameter - for parameter in space.searchables.values() + for parameter in pipeline_space.searchables.values() if parameter.prior is not None ] raise ValueError( @@ -1093,7 +1357,7 @@ def bayesian_optimization( ) return _bo( - pipeline_space=space, + pipeline_space=pipeline_space, initial_design_size=initial_design_size, cost_aware=cost_aware, device=device, @@ -1105,7 +1369,7 @@ def bayesian_optimization( def pibo( - space: SearchSpace, + pipeline_space: SearchSpace | PipelineSpace, *, initial_design_size: int | Literal["ndim"] = "ndim", cost_aware: bool | Literal["log"] = False, @@ -1127,7 +1391,7 @@ def pibo( has. Args: - space: The search space to sample from. + pipeline_space: The search space to sample from. initial_design_size: Number of samples used before using the surrogate model. If "ndim", it will use the number of parameters in the search space. cost_aware: Whether to consider reported "cost" from configurations in decision @@ -1135,27 +1399,37 @@ def pibo( they cost, incentivising the optimizer to explore cheap, good performing configurations. This amount is modified over time. If "log", the cost will be log-transformed before being used. - !!! warning + !!! warning "Cost aware" + + If using `cost`, cost must be provided in the reports of the trials. - If using `cost`, cost must be provided in the reports of the trials. device: Device to use for the optimization. sample_prior_first: Whether to sample the prior configuration first. ignore_fidelity: Whether to ignore the fidelity parameter when sampling. In this case, the max fidelity is always used. """ - if all(parameter.prior is None for parameter in space.searchables.values()): + if isinstance(pipeline_space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(pipeline_space) + if converted_space is not None: + pipeline_space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) + if all(parameter.prior is None for parameter in pipeline_space.searchables.values()): logger.warning( "Warning: PiBO was called without any priors - using uniform priors on all" " parameters.\nConsider using Bayesian Optimization instead." ) - if ignore_fidelity and space.fidelity is None: + if ignore_fidelity and pipeline_space.fidelity is None: logger.warning( "Warning: You are using ignore_fidelity, but no fidelity is defined in the" " search space. Consider setting ignore_fidelity to False." ) return _bo( - pipeline_space=space, + pipeline_space=pipeline_space, initial_design_size=initial_design_size, cost_aware=cost_aware, device=device, @@ -1248,7 +1522,7 @@ class CustomOptimizer: kwargs: Mapping[str, Any] = field(default_factory=dict) initialized: bool = False - def create(self, space: SearchSpace) -> AskFunction: + def create(self, space: SearchSpace | PipelineSpace) -> AskFunction: assert not self.initialized, "Custom optimizer already initialized." return self.optimizer(space, **self.kwargs) # type: ignore @@ -1276,10 +1550,344 @@ def custom( ) -PredefinedOptimizers: Mapping[ - str, - Callable[Concatenate[SearchSpace, ...], AskFunction], -] = { +def complex_random_search( + pipeline_space: PipelineSpace, + *, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, +) -> NePSComplexRandomSearch: + """A complex random search algorithm that samples configurations uniformly at random, + but allows for more complex sampling strategies. + + Args: + pipeline_space: The search space to sample from. + ignore_fidelity: Whether to ignore the fidelity parameter when sampling. + If `True`, the algorithm will sample the fidelity like a normal parameter. + If set to `"highest_fidelity"`, it will always sample at the highest_fidelity. + Raises: + ValueError: If the pipeline has fidelity attributes and `ignore_fidelity` is + set to `False`. Complex random search does not support fidelities by default. + """ + + if pipeline_space.fidelity_attrs and ignore_fidelity is False: + raise ValueError( + "Complex Random Search does not support fidelities by default." + "Consider using `ignore_fidelity=True` or `highest_fidelity`" + "to always sample at max fidelity." + ) + if not pipeline_space.fidelity_attrs and ignore_fidelity is not False: + logger.warning( + "You are using ignore_fidelity, but no fidelity is defined in the" + " search space. Consider setting ignore_fidelity to False." + ) + + return NePSComplexRandomSearch( + pipeline=pipeline_space, + ignore_fidelity=ignore_fidelity, + ) + + +def neps_random_search( + pipeline_space: PipelineSpace, + *, + use_priors: bool = False, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, +) -> NePSRandomSearch: + """A simple random search algorithm that samples configurations uniformly at random. + + Args: + pipeline_space: The search space to sample from. + use_priors: Whether to use priors when sampling. + If `True`, the algorithm will sample from the prior distribution + defined in the search space. + ignore_fidelity: Whether to ignore the fidelity parameter when sampling. + If `True`, the algorithm will sample the fidelity like a normal parameter. + If set to `"highest_fidelity"`, it will always sample at the highest_fidelity. + Raises: + ValueError: If the pipeline space has fidelity attributes and `ignore_fidelity` is + set to `False`. Random search does not support fidelities by default. + """ + + if pipeline_space.fidelity_attrs and ignore_fidelity is False: + raise ValueError( + "Random Search does not support fidelities by default." + "Consider using `ignore_fidelity=True` or `highest_fidelity`" + "to always sample at max fidelity." + ) + if not pipeline_space.fidelity_attrs and ignore_fidelity is not False: + logger.warning( + "You are using ignore_fidelity, but no fidelity is defined in the" + " search space. Consider setting ignore_fidelity to False." + ) + if use_priors and not pipeline_space.has_priors(): + logger.warning( + "You have set use_priors=True, but no priors are defined in the search space." + ) + + return NePSRandomSearch( + pipeline=pipeline_space, use_priors=use_priors, ignore_fidelity=ignore_fidelity + ) + + +def _neps_bracket_optimizer( # noqa: C901, PLR0915 + pipeline_space: PipelineSpace, + *, + bracket_type: Literal["successive_halving", "hyperband", "asha", "async_hb"], + eta: int, + sampler: Literal["priorband", "uniform", "prior", "local_and_incumbent"], + sample_prior_first: bool | Literal["highest_fidelity"], + early_stopping_rate: int | None, + sampler_kwargs: dict[str, Any] | None = None, +) -> _NePSBracketOptimizer: + fidelity_attrs = pipeline_space.fidelity_attrs + + if len(fidelity_attrs.items()) != 1: + raise ValueError( + "Exactly one fidelity should be defined in the pipeline space." + f"\nGot: {fidelity_attrs!r}" + ) + + fidelity_name, fidelity_obj = next(iter(fidelity_attrs.items())) + fidelity_name = NepsCompatConverter._ENVIRONMENT_PREFIX + fidelity_name + + if sample_prior_first not in (True, False, "highest_fidelity"): + raise ValueError( + "sample_prior_first should be either True, False or 'highest_fidelity'" + ) + + from neps.optimizers.utils import brackets + + # Determine the strategy for creating brackets for sampling + create_brackets: Callable[[pd.DataFrame], Sequence[Bracket] | Bracket] + match bracket_type: + case "successive_halving": + assert early_stopping_rate is not None + rung_to_fidelity, rung_sizes = brackets.calculate_sh_rungs( + bounds=(fidelity_obj.lower, fidelity_obj.upper), + eta=eta, + early_stopping_rate=early_stopping_rate, + ) + create_brackets = partial( + brackets.Sync.create_repeating, + rung_sizes=rung_sizes, + ) + rung_fidelity_str = "\n".join( + f"{k}: {v}" for k, v in rung_to_fidelity.items() + ) + logging.info(f"Successive Halving Rung to Fidelity:\n{rung_fidelity_str}") + rung_sizes_str = "\n".join(f"{k}: {v}" for k, v in rung_sizes.items()) + logging.info(f"Successive Halving Rung Sizes:\n{rung_sizes_str}") + + case "hyperband": + assert early_stopping_rate is None + rung_to_fidelity, bracket_layouts = brackets.calculate_hb_bracket_layouts( + bounds=(fidelity_obj.lower, fidelity_obj.upper), + eta=eta, + ) + create_brackets = partial( + brackets.Hyperband.create_repeating, + bracket_layouts=bracket_layouts, + ) + rung_fidelity_str = "\n".join( + f"Rung {k}: Fidelity >= {v}" for k, v in rung_to_fidelity.items() + ) + logging.info(f"Hyperband Rung to Fidelity:\n{rung_fidelity_str}") + bracket_layouts_str = "\n\n".join( + f"Bracket {i}\n" + + "\n".join([f"At Rung {k}: {v} configs" for k, v in bracket.items()]) + for i, bracket in enumerate(bracket_layouts) + ) + logging.info(f"Hyperband Bracket Layouts:\n{bracket_layouts_str}") + + case "asha": + assert early_stopping_rate is not None + rung_to_fidelity, _rung_sizes = brackets.calculate_sh_rungs( + bounds=(fidelity_obj.lower, fidelity_obj.upper), + eta=eta, + early_stopping_rate=early_stopping_rate, + ) + create_brackets = partial( + brackets.Async.create, + rungs=list(rung_to_fidelity), + eta=eta, + ) + rung_fidelity_str = "\n".join( + f"{k}: {v}" for k, v in rung_to_fidelity.items() + ) + logging.info(f"ASHA Rung to Fidelity:\n{rung_fidelity_str}") + rung_sizes_str = "\n".join(f"{k}: {v}" for k, v in _rung_sizes.items()) + logging.info(f"ASHA Rung Sizes:\n{rung_sizes_str}") + + case "async_hb": + assert early_stopping_rate is None + rung_to_fidelity, bracket_layouts = brackets.calculate_hb_bracket_layouts( + bounds=(fidelity_obj.lower, fidelity_obj.upper), + eta=eta, + ) + # We don't care about the capacity of each bracket, we need the rung layout + bracket_rungs = [list(bracket.keys()) for bracket in bracket_layouts] + create_brackets = partial( + brackets.AsyncHyperband.create, + bracket_rungs=bracket_rungs, + eta=eta, + ) + rung_fidelity_str = "\n".join( + f"Rung {k}: Fidelity >= {v}" for k, v in rung_to_fidelity.items() + ) + logging.info(f"Async HB Rung to Fidelity:\n{rung_fidelity_str}") + bracket_rungs_str = "\n\n".join( + f"Bracket {i}\n" + "\n".join([f"At Rung {k}" for k in bracket]) + for i, bracket in enumerate(bracket_rungs) + ) + logging.info(f"Async Hyperband Bracket Rungs:\n{bracket_rungs_str}") + case _: + raise ValueError(f"Unknown bracket type: {bracket_type}") + + _sampler_kwargs = sampler_kwargs or {} + _sampler: NePSPriorBandSampler | DomainSampler | NePSLocalPriorIncumbentSampler + match sampler: + case "priorband": + _sampler = NePSPriorBandSampler( + space=pipeline_space, + eta=eta, + early_stopping_rate=( + early_stopping_rate if early_stopping_rate is not None else 0 + ), + fid_bounds=(fidelity_obj.lower, fidelity_obj.upper), + **_sampler_kwargs, + ) + case "local_and_incumbent": + _sampler = NePSLocalPriorIncumbentSampler( + space=pipeline_space, + **_sampler_kwargs, + ) + case "uniform": + _sampler = RandomSampler({}) + case "prior": + _sampler = PriorOrFallbackSampler( + fallback_sampler=RandomSampler({}), always_use_prior=False + ) + case _: + raise ValueError(f"Unknown sampler: {sampler}") + + return _NePSBracketOptimizer( + space=pipeline_space, + eta=eta, + rung_to_fid=rung_to_fidelity, + sampler=_sampler, + sample_prior_first=sample_prior_first, + create_brackets=create_brackets, + fid_name=fidelity_name, + ) + + +def neps_priorband( + pipeline_space: PipelineSpace, + *, + inc_ratio: float = 0.9, + eta: int = 3, + sample_prior_first: bool | Literal["highest_fidelity"] = False, + base: Literal["successive_halving", "hyperband", "asha", "async_hb"] = "hyperband", +) -> _NePSBracketOptimizer: + """Create a PriorBand optimizer for the given pipeline space. + + Args: + pipeline_space: The pipeline space to optimize over. + eta: The eta parameter for the algorithm. + sample_prior_first: Whether to sample the prior first. + If set to `"highest_fidelity"`, the prior will be sampled at the + highest_fidelity, otherwise at the lowest fidelity. + base: The type of bracket optimizer to use. One of: + - "successive_halving" + - "hyperband" + - "asha" + - "async_hb" + Returns: + An instance of _BracketOptimizer configured for PriorBand sampling. + """ + if not pipeline_space.has_priors(): + logger.warning( + "Warning: No priors are defined in the search space, priorband will sample" + " uniformly. Consider using hyperband instead." + ) + return _neps_bracket_optimizer( + pipeline_space=pipeline_space, + bracket_type=base, + eta=eta, + sampler="priorband", + sample_prior_first=sample_prior_first, + early_stopping_rate=0 if base in ("successive_halving", "asha") else None, + sampler_kwargs={"inc_ratio": inc_ratio}, + ) + + +def neps_regularized_evolution( + pipeline_space: PipelineSpace, + *, + population_size: int = 20, + tournament_size: int = 5, + use_priors: bool = True, + mutation_type: float | Literal["mutate_best", "crossover_top_2"] = 0.5, + n_mutations: int | Literal["random", "half"] | None = "random", + n_forgets: int | Literal["random", "half"] | None = None, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, +) -> NePSRegularizedEvolution: + return NePSRegularizedEvolution( + pipeline=pipeline_space, + population_size=population_size, + tournament_size=tournament_size, + use_priors=use_priors, + mutation_type=mutation_type, + n_mutations=n_mutations, + n_forgets=n_forgets, + ignore_fidelity=ignore_fidelity, + ) + + +def neps_local_and_incumbent( + pipeline_space: PipelineSpace, + *, + local_prior: dict[str, Any], + inc_takeover_mode: Literal[0, 1, 2, 3] = 0, + random_ratio: float = 0.0, + eta: int = 3, + base: Literal["successive_halving", "hyperband", "asha", "async_hb"] = "hyperband", +) -> _NePSBracketOptimizer: + """Create a LocalAndIncumbent optimizer for the given pipeline space. + + Args: + pipeline_space: The pipeline space to optimize over. + local_prior: The local prior to use for sampling. + inc_takeover_mode: The incumbent takeover mode. + eta: The eta parameter for the algorithm. + base: The type of bracket optimizer to use. One of: + - "successive_halving" + - "hyperband" + - "asha" + - "async_hb" + Returns: + An instance of _BracketOptimizer configured for LocalAndIncumbent sampling. + """ + if pipeline_space.has_priors(): + logger.warning( + "Warning: Priors are defined in the search space, but LocalAndIncumbent does" + " not use them." + ) + return _neps_bracket_optimizer( + pipeline_space=pipeline_space, + bracket_type=base, + eta=eta, + sampler="local_and_incumbent", + sample_prior_first=False, + early_stopping_rate=0 if base in ("successive_halving", "asha") else None, + sampler_kwargs={ + "local_prior": local_prior, + "inc_takeover_mode": inc_takeover_mode, + "random_ratio": random_ratio, + }, + ) + + +PredefinedOptimizers: Mapping[str, Any] = { f.__name__: f for f in ( bayesian_optimization, @@ -1295,6 +1903,12 @@ def custom( async_hb, priorband, primo, + neps_random_search, + complex_random_search, + neps_priorband, + neps_hyperband, + neps_regularized_evolution, + neps_local_and_incumbent, ) } @@ -1312,4 +1926,10 @@ def custom( "grid_search", "ifbo", "primo", + "neps_random_search", + "complex_random_search", + "neps_priorband", + "neps_hyperband", + "neps_regularized_evolution", + "neps_local_and_incumbent", ] diff --git a/neps/optimizers/ask_and_tell.py b/neps/optimizers/ask_and_tell.py index b3a02fbcd..eacce1268 100644 --- a/neps/optimizers/ask_and_tell.py +++ b/neps/optimizers/ask_and_tell.py @@ -72,6 +72,7 @@ def evaluate(config): import time from collections.abc import Mapping from dataclasses import dataclass, field +from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, overload from neps.optimizers.optimizer import AskFunction, SampledConfig @@ -79,7 +80,6 @@ def evaluate(config): if TYPE_CHECKING: from neps.state.optimizer import BudgetInfo - from neps.state.pipeline_eval import EvaluatePipelineReturn def _default_worker_name() -> str: @@ -180,6 +180,7 @@ def tell_custom( previous_trial_id: str | None = None, worker_id: str | None = None, traceback_str: str | None = None, + location: Path | None = None, ) -> Trial: """Report a custom configuration and result to the optimizer. @@ -207,6 +208,8 @@ def tell_custom( metadata if you need. traceback_str: The traceback of any error, only to fill in metadata if you need. + location: The location of the configuration, if any. This will be saved + in the created trial's metadata. Returns: The trial object that was created. You can find the report @@ -232,7 +235,7 @@ def tell_custom( # Just go through the motions of the trial life-cycle trial = Trial.new( trial_id=config_id, - location="", + location=str(location.resolve()) if location else "", config=config, previous_trial=previous_trial_id, previous_trial_location="", @@ -269,8 +272,8 @@ def tell( """Report the result of an evaluation back to the optimizer. Args: - config_id: The id of the configuration you got from - [`ask()`][neps.optimizers.ask_and_tell.AskAndTell.ask]. + trial: The trial to report the result for. This can be either + the trial id (a string) or the trial object itself. result: The result of the evaluation. This can be an exception, a float, or a mapping of values, similar to that which you would return from `evaluate_pipeline` when your normally diff --git a/neps/optimizers/bayesian_optimization.py b/neps/optimizers/bayesian_optimization.py index 1160a9fb2..71150cb5e 100644 --- a/neps/optimizers/bayesian_optimization.py +++ b/neps/optimizers/bayesian_optimization.py @@ -23,6 +23,8 @@ ) from neps.optimizers.optimizer import ImportedConfig, SampledConfig from neps.optimizers.utils.initial_design import make_initial_design +from neps.space.neps_spaces.neps_space import convert_neps_to_classic_search_space +from neps.space.neps_spaces.parameters import PipelineSpace if TYPE_CHECKING: from neps.sampling import Prior @@ -65,7 +67,7 @@ def _pibo_exp_term( class BayesianOptimization: """Uses `botorch` as an engine for doing bayesian optimiziation.""" - space: SearchSpace + space: SearchSpace | PipelineSpace """The search space to use.""" encoder: ConfigEncoder @@ -95,6 +97,16 @@ def __call__( # noqa: C901, PLR0912, PLR0915 # noqa: C901, PLR0912 budget_info: BudgetInfo | None = None, n: int | None = None, ) -> SampledConfig | list[SampledConfig]: + if isinstance(self.space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(self.space) + if converted_space is not None: + self.space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) + # If fidelities exist, sample from them as normal # This is a bit of a hack, as we set them to max fidelity # afterwards, but we need the complete space to sample diff --git a/neps/optimizers/bracket_optimizer.py b/neps/optimizers/bracket_optimizer.py index c357b8419..793d0a421 100644 --- a/neps/optimizers/bracket_optimizer.py +++ b/neps/optimizers/bracket_optimizer.py @@ -26,6 +26,8 @@ get_trial_config_unique_key, ) from neps.sampling.samplers import Sampler +from neps.space.neps_spaces.neps_space import convert_neps_to_classic_search_space +from neps.space.neps_spaces.parameters import PipelineSpace from neps.utils.common import disable_warnings if TYPE_CHECKING: @@ -219,7 +221,7 @@ class BracketOptimizer: `"successive_halving"`, `"asha"`, `"hyperband"`, etc. """ - space: SearchSpace + space: SearchSpace | PipelineSpace """The pipeline space to optimize over.""" encoder: ConfigEncoder @@ -258,12 +260,22 @@ class BracketOptimizer: fid_name: str """The name of the fidelity in the space.""" - def __call__( # noqa: C901, PLR0912 + def __call__( # noqa: C901, PLR0912, PLR0915 self, trials: Mapping[str, Trial], budget_info: BudgetInfo | None, n: int | None = None, ) -> SampledConfig | list[SampledConfig]: + if isinstance(self.space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(self.space) + if converted_space is not None: + self.space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) + assert n is None, "paramter n should be not None" space = self.space parameters = space.searchables @@ -409,7 +421,7 @@ def import_trials( f"{list(rung_to_fid.values())}. Skipping config: {config}" ) continue - # create a unique key for the config without the fidelity + # create a unique key for the config without the fidelity config_key = get_trial_config_unique_key( config=config, fid_name=self.fid_name ) diff --git a/neps/optimizers/ifbo.py b/neps/optimizers/ifbo.py index 6ab898a95..66dce4a4a 100755 --- a/neps/optimizers/ifbo.py +++ b/neps/optimizers/ifbo.py @@ -18,7 +18,9 @@ from neps.optimizers.utils.initial_design import make_initial_design from neps.optimizers.utils.util import get_trial_config_unique_key from neps.sampling import Prior, Sampler -from neps.space import ConfigEncoder, Domain, Float, Integer, SearchSpace +from neps.space import ConfigEncoder, Domain, HPOFloat, HPOInteger, SearchSpace +from neps.space.neps_spaces.neps_space import convert_neps_to_classic_search_space +from neps.space.neps_spaces.parameters import PipelineSpace if TYPE_CHECKING: from neps.state import BudgetInfo, Trial @@ -73,10 +75,10 @@ def _adjust_space_to_match_stepsize( r = x - n * step_size new_lower = fidelity.lower + r - new_fid: Float | Integer + new_fid: HPOFloat | HPOInteger match fidelity: - case Float(): - new_fid = Float( + case HPOFloat(): + new_fid = HPOFloat( lower=float(new_lower), upper=float(fidelity.upper), log=fidelity.log, @@ -84,8 +86,8 @@ def _adjust_space_to_match_stepsize( is_fidelity=True, prior_confidence=fidelity.prior_confidence, ) - case Integer(): - new_fid = Integer( + case HPOInteger(): + new_fid = HPOInteger( lower=int(new_lower), upper=int(fidelity.upper), log=fidelity.log, @@ -107,7 +109,7 @@ class IFBO: * Github: https://github.com/automl/ifBO/tree/main """ - space: SearchSpace + space: SearchSpace | PipelineSpace """The entire search space for the pipeline.""" encoder: ConfigEncoder @@ -140,6 +142,15 @@ def __call__( budget_info: BudgetInfo | None = None, n: int | None = None, ) -> SampledConfig | list[SampledConfig]: + if isinstance(self.space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(self.space) + if converted_space is not None: + self.space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) assert self.space.fidelity is not None fidelity_name, fidelity = self.space.fidelity parameters = self.space.searchables @@ -297,6 +308,7 @@ def import_trials( return [] imported: list[ImportedConfig] = [] + assert isinstance(self.space, SearchSpace) assert self.space.fidelity is not None fidelity_name, fidelity = self.space.fidelity budget_index_domain = Domain.indices(self.n_fidelity_bins + 1) diff --git a/neps/optimizers/models/ftpfn.py b/neps/optimizers/models/ftpfn.py index 72e88ce7a..b69337be8 100644 --- a/neps/optimizers/models/ftpfn.py +++ b/neps/optimizers/models/ftpfn.py @@ -10,7 +10,7 @@ from neps.sampling import Prior, Sampler if TYPE_CHECKING: - from neps.space import ConfigEncoder, Domain, Float, Integer + from neps.space import ConfigEncoder, Domain, HPOFloat, HPOInteger from neps.state.trial import Trial @@ -106,7 +106,7 @@ def _cast_tensor_shapes(x: torch.Tensor) -> torch.Tensor: def encode_ftpfn( trials: Mapping[str, Trial], - fid: tuple[str, Integer | Float], + fid: tuple[str, HPOInteger | HPOFloat], budget_domain: Domain, encoder: ConfigEncoder, *, @@ -131,7 +131,6 @@ def encode_ftpfn( Args: trials: The trials to encode encoder: The encoder to use - space: The search space budget_domain: The domain to use for the budgets of the FTPFN device: The device to use dtype: The dtype to use @@ -169,12 +168,14 @@ def encode_ftpfn( # We could possibly include some bounded transform to assert this. minimize_ys = torch.tensor( [ - pending_value - if trial.report is None - else ( - error_value - if trial.report.objective_to_minimize is None - else trial.report.objective_to_minimize + ( + pending_value + if trial.report is None + else ( + error_value + if trial.report.objective_to_minimize is None + else trial.report.objective_to_minimize + ) ) for trial in trials.values() ], diff --git a/neps/optimizers/models/gp.py b/neps/optimizers/models/gp.py index 586ba371e..a70a235bb 100644 --- a/neps/optimizers/models/gp.py +++ b/neps/optimizers/models/gp.py @@ -248,7 +248,6 @@ def encode_trials_for_gp( Args: trials: The trials to encode. - space: The search space. encoder: The encoder to use. If `None`, one will be created. device: The device to use. diff --git a/neps/optimizers/neps_bracket_optimizer.py b/neps/optimizers/neps_bracket_optimizer.py new file mode 100644 index 000000000..9e7d6dc6b --- /dev/null +++ b/neps/optimizers/neps_bracket_optimizer.py @@ -0,0 +1,278 @@ +"""This module provides multi-fidelity optimizers for NePS spaces. +It implements a bracket-based optimization strategy that samples configurations +from a prior band, allowing for efficient exploration of the search space. +It supports different bracket types such as successive halving, hyperband, ASHA, +and async hyperband, and can sample configurations at different fidelity levels. +""" + +from __future__ import annotations + +import copy +import logging +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +import pandas as pd + +import neps.optimizers.bracket_optimizer as standard_bracket_optimizer +from neps.optimizers.neps_local_and_incumbent import NePSLocalPriorIncumbentSampler +from neps.optimizers.neps_priorband import NePSPriorBandSampler +from neps.optimizers.optimizer import ImportedConfig, SampledConfig +from neps.optimizers.utils.brackets import PromoteAction, SampleAction +from neps.optimizers.utils.util import ( + get_config_key_to_id_mapping, + get_trial_config_unique_key, +) +from neps.space.neps_spaces import neps_space +from neps.space.neps_spaces.sampling import ( + DomainSampler, + OnlyPredefinedValuesSampler, + PriorOrFallbackSampler, + RandomSampler, +) + +if TYPE_CHECKING: + from neps.optimizers.utils.brackets import Bracket + from neps.space.neps_spaces.parameters import PipelineSpace + from neps.state.optimizer import BudgetInfo + from neps.state.pipeline_eval import UserResultDict + from neps.state.trial import Trial + + +logger = logging.getLogger(__name__) + + +@dataclass +class _NePSBracketOptimizer: + """The pipeline space to optimize over.""" + + space: PipelineSpace + + """Whether or not to sample the prior first. + + If set to `"highest_fidelity"`, the prior will be sampled at the highest fidelity, + otherwise at the lowest fidelity. + """ + sample_prior_first: bool | Literal["highest_fidelity"] + + """The eta parameter for the algorithm.""" + eta: int + + """The mapping from rung to fidelity value.""" + rung_to_fid: Mapping[int, int | float] + + """A function that creates the brackets from the table of trials.""" + create_brackets: Callable[[pd.DataFrame], Sequence[Bracket] | Bracket] + + """The sampler used to generate new trials.""" + sampler: NePSPriorBandSampler | DomainSampler | NePSLocalPriorIncumbentSampler + + """The name of the fidelity in the space.""" + fid_name: str + + def __call__( # noqa: C901, PLR0912 + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: + assert n is None, "TODO" + + # If we have no trials, we either go with the prior or just a sampled config + if len(trials) == 0: + match self.sample_prior_first: + case "highest_fidelity": # fid_max + config = self._sample_prior(fidelity_level="max") + rung = max(self.rung_to_fid) + return SampledConfig(id=f"1_rung_{rung}", config=config) + case True: # fid_min + config = self._sample_prior(fidelity_level="min") + rung = min(self.rung_to_fid) + return SampledConfig(id=f"1__rung_{rung}", config=config) + case False: + pass + + table = standard_bracket_optimizer.trials_to_table(trials=trials) + + if len(table) == 0: # noqa: SIM108 + # Nothing there, this sample will be the first + nxt_id = 1 + else: + # One plus the maximum current id in the table index + nxt_id = table.index.get_level_values("id").max() + 1 # type: ignore + + # We don't want the first highest fidelity sample ending + # up in a bracket + if self.sample_prior_first == "highest_fidelity": + table = table.iloc[1:] + + # Get and execute the next action from our brackets that are not pending or done + assert isinstance(table, pd.DataFrame) + brackets = self.create_brackets(table) + + if not isinstance(brackets, Sequence): + brackets = [brackets] + + next_action = next( + ( + action + for bracket in brackets + if (action := bracket.next()) not in ("done", "pending") + ), + None, + ) + + if next_action is None: + raise RuntimeError( + f"{self.__class__.__name__} never got a 'sample' or 'promote' action!" + f" This likely means the implementation of {self.create_brackets}" + " is incorrect and should have provded enough brackets, where at" + " least one of them should have requested another sample." + f"\nBrackets:\n{brackets}" + ) + + match next_action: + # The bracket would like us to promote a configuration + case PromoteAction(config=config, id=config_id, new_rung=new_rung): + config = self._convert_to_another_rung(config=config, rung=new_rung) + return SampledConfig( + id=f"{config_id}_rung_{new_rung}", + config=config, + previous_config_id=f"{config_id}_rung_{new_rung - 1}", + ) + + # We need to sample for a new rung. + case SampleAction(rung=rung): + if isinstance(self.sampler, NePSPriorBandSampler): + config = self.sampler.sample_config(table, rung=rung) + elif isinstance(self.sampler, NePSLocalPriorIncumbentSampler): + config = self.sampler.sample_config(table) + elif isinstance(self.sampler, DomainSampler): + environment_values = {} + fidelity_attrs = self.space.fidelity_attrs + assert len(fidelity_attrs) == 1, "TODO: [lum]" + for fidelity_name, _fidelity_obj in fidelity_attrs.items(): + environment_values[fidelity_name] = self.rung_to_fid[rung] + _, resolution_context = neps_space.resolve( + self.space, + domain_sampler=self.sampler, + environment_values=environment_values, + ) + config = neps_space.NepsCompatConverter.to_neps_config( # type: ignore[assignment] + resolution_context + ) + config = dict(**config) + config = self._convert_to_another_rung(config=config, rung=rung) + return SampledConfig( + id=f"{nxt_id}_rung_{rung}", + config=config, + ) + + case _: + raise RuntimeError(f"Unknown bracket action: {next_action}") + + def _sample_prior( + self, + fidelity_level: Literal["min"] | Literal["max"], + ) -> dict[str, Any]: + # TODO: [lum] have a CenterSampler as fallback, not Random + _try_always_priors_sampler = PriorOrFallbackSampler( + fallback_sampler=RandomSampler(predefined_samplings={}), + always_use_prior=True, + ) + + _environment_values = {} + _fidelity_attrs = self.space.fidelity_attrs + for fidelity_name, fidelity_obj in _fidelity_attrs.items(): + if fidelity_level == "max": + _environment_values[fidelity_name] = fidelity_obj.upper + elif fidelity_level == "min": + _environment_values[fidelity_name] = fidelity_obj.lower + else: + raise ValueError(f"Invalid fidelity level {fidelity_level}") + + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=_try_always_priors_sampler, + environment_values=_environment_values, + ) + + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) + + def _convert_to_another_rung( + self, + config: Mapping[str, Any], + rung: int, + ) -> dict[str, Any]: + data = neps_space.NepsCompatConverter.from_neps_config(config=config) + + _environment_values = {} + _fidelity_attrs = self.space.fidelity_attrs + assert len(_fidelity_attrs) == 1, "TODO: [lum]" + for fidelity_name, _fidelity_obj in _fidelity_attrs.items(): + _environment_values[fidelity_name] = self.rung_to_fid[rung] + + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=OnlyPredefinedValuesSampler( + predefined_samplings=data.predefined_samplings, + ), + environment_values=_environment_values, + ) + + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) + + def import_trials( + self, + external_evaluations: Sequence[tuple[Mapping[str, Any], UserResultDict]], + trials: Mapping[str, Trial], + ) -> list[ImportedConfig]: + rung_to_fid = self.rung_to_fid + + # Use trials_to_table to get all used config IDs + table = standard_bracket_optimizer.trials_to_table(trials) + used_ids = set(table.index.get_level_values("id").tolist()) + + imported_configs = [] + config_to_id = get_config_key_to_id_mapping(table=table, fid_name=self.fid_name) + + for config, result in external_evaluations: + fid_value = config[self.fid_name] + # create a unique key for the config without the fidelity + config_key = get_trial_config_unique_key( + config=config, fid_name=self.fid_name + ) + # Assign id if not already assigned + if config_key not in config_to_id: + next_id = max(used_ids, default=0) + 1 + config_to_id[config_key] = next_id + used_ids.add(next_id) + else: + existing_id = config_to_id[config_key] + # check if the other config with same key has the same fidelity + try: + existing_config = table.xs(existing_id, level="id")["config"].iloc[0] + if existing_config[self.fid_name] == config[self.fid_name]: + logger.warning( + f"Duplicate configuration with same fidelity found: {config}" + ) + continue + except KeyError: + pass + + config_id = config_to_id[config_key] + + # Find the rung corresponding to the fidelity value in config + rung = max((r for r, f in rung_to_fid.items() if f <= fid_value)) + trial_id = f"{config_id}_rung_{rung}" + imported_configs.append( + ImportedConfig( + id=trial_id, + config=copy.deepcopy(config), + result=copy.deepcopy(result), + ) + ) + return imported_configs diff --git a/neps/optimizers/neps_local_and_incumbent.py b/neps/optimizers/neps_local_and_incumbent.py new file mode 100644 index 000000000..44461a7c4 --- /dev/null +++ b/neps/optimizers/neps_local_and_incumbent.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import logging +import random +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +import neps.space.neps_spaces.sampling +from neps.space.neps_spaces import neps_space, sampling + +if TYPE_CHECKING: + import pandas as pd + + from neps.space.neps_spaces.parameters import PipelineSpace + + +@dataclass +class NePSLocalPriorIncumbentSampler: + """Implement a sampler that samples from the incumbent.""" + + space: PipelineSpace + """The pipeline space to optimize over.""" + + random_ratio: float = 0.0 + """The ratio of random sampling vs incumbent sampling.""" + + local_prior: dict[str, Any] | None = None + """The local prior configuration.""" + + inc_takeover_mode: Literal[0, 1, 2, 3] = 0 + """The incumbent takeover mode. + 0: Always mutate the first config. + 1: Use the global incumbent. + 2: Crossover between global incumbent and first config. + 3: Choose randomly between 0, 1, and 2. + """ + + def sample_config(self, table: pd.DataFrame) -> dict[str, Any]: # noqa: C901 + """Sample a configuration based on the PriorBand algorithm. + + Args: + table (pd.DataFrame): The table containing the configurations and their + performance. + + Returns: + dict[str, Any]: A sampled configuration. + """ + + completed: pd.DataFrame = table[table["perf"].notna()] # type: ignore + if completed.empty: + logging.warning("No local prior found. Sampling randomly from the space.") + return ( + self.local_prior + if self.local_prior is not None + else self._sample_random() + ) + + # If no local prior is given, save the first config as the local prior + if self.local_prior is None: + first_config = completed.iloc[0]["config"] + assert isinstance(first_config, dict) + self.local_prior = first_config + + # Get the incumbent configuration + inc_config = completed.loc[completed["perf"].idxmin()]["config"] + first_config = self.local_prior + assert isinstance(inc_config, dict) + + # Decide whether to sample randomly or from the incumbent + if random.random() < self.random_ratio: + return self._sample_random() + + match self.inc_takeover_mode: + case 0: + # Always mutate the first config. + new_config = self._mutate_inc(inc_config=first_config) + case 1: + # Use the global incumbent. + new_config = self._mutate_inc(inc_config=inc_config) + case 2: + # Crossover between global incumbent and first config. + new_config = self._crossover_incs( + inc_config=inc_config, + first_config=first_config, + ) + case 3: + # Choose randomly between 0, 1, and 2. + match random.randint(0, 2): + case 0: + new_config = self._mutate_inc(inc_config=first_config) + case 1: + new_config = self._mutate_inc(inc_config=inc_config) + case 2: + new_config = self._crossover_incs( + inc_config=inc_config, + first_config=first_config, + ) + case _: + raise ValueError( + "This should never happen. Only for type checking." + ) + case _: + raise ValueError(f"Invalid inc_takeover_mode: {self.inc_takeover_mode}") + return new_config + + def _sample_random(self) -> dict[str, Any]: + # Sample randomly from the space + _environment_values = {} + _fidelity_attrs = self.space.fidelity_attrs + for fidelity_name, fidelity_obj in _fidelity_attrs.items(): + _environment_values[fidelity_name] = fidelity_obj.upper + + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=sampling.RandomSampler({}), + environment_values=_environment_values, + ) + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) + + def _mutate_inc(self, inc_config: dict[str, Any]) -> dict[str, Any]: + _environment_values = {} + _fidelity_attrs = self.space.fidelity_attrs + for fidelity_name, fidelity_obj in _fidelity_attrs.items(): + _environment_values[fidelity_name] = fidelity_obj.upper + + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=neps.space.neps_spaces.sampling.MutatateUsingCentersSampler( + predefined_samplings=inc_config, + n_mutations=max(1, random.randint(1, int(len(inc_config) / 2))), + ), + environment_values=_environment_values, + ) + + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) + + def _crossover_incs( + self, inc_config: dict[str, Any], first_config: dict[str, Any] + ) -> dict[str, Any]: + _environment_values = {} + _fidelity_attrs = self.space.fidelity_attrs + for fidelity_name, fidelity_obj in _fidelity_attrs.items(): + _environment_values[fidelity_name] = fidelity_obj.upper + + # Crossover between the best two trials' configs to create a new config. + try: + crossover_sampler = sampling.CrossoverByMixingSampler( + predefined_samplings_1=inc_config, + predefined_samplings_2=first_config, + prefer_first_probability=0.5, + ) + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=crossover_sampler, + environment_values=_environment_values, + ) + except sampling.CrossoverNotPossibleError: + # A crossover was not possible for them. Increase configs and try again. + # If we have tried all crossovers, mutate the best instead. + # Mutate 50% of the top trial's config. + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=sampling.MutatateUsingCentersSampler( + predefined_samplings=inc_config, + n_mutations=max(1, int(len(inc_config) / 2)), + ), + environment_values=_environment_values, + ) + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) diff --git a/neps/optimizers/neps_priorband.py b/neps/optimizers/neps_priorband.py new file mode 100644 index 000000000..86c824570 --- /dev/null +++ b/neps/optimizers/neps_priorband.py @@ -0,0 +1,214 @@ +"""PriorBand Sampler for NePS Optimizers. +This sampler implements the PriorBand algorithm, which is a sampling strategy +that combines prior knowledge with random sampling to efficiently explore the search +space. It uses a combination of prior sampling, incumbent mutation, and random sampling +based on the fidelity bounds and SH bracket. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import numpy as np + +import neps.space.neps_spaces.sampling +from neps.optimizers.utils import brackets +from neps.space.neps_spaces import neps_space + +if TYPE_CHECKING: + import pandas as pd + + from neps.space.neps_spaces.parameters import PipelineSpace + + +@dataclass +class NePSPriorBandSampler: + """Implement a sampler based on PriorBand.""" + + space: PipelineSpace + """The pipeline space to optimize over.""" + + eta: int + """The eta value to use for the SH bracket.""" + + early_stopping_rate: int + """The early stopping rate to use for the SH bracket.""" + + fid_bounds: tuple[int, int] | tuple[float, float] + """The fidelity bounds.""" + + inc_ratio: float = 0.9 + """The ratio of the incumbent (vs. prior) in the sampling distribution.""" + + def sample_config(self, table: pd.DataFrame, rung: int) -> dict[str, Any]: + """Sample a configuration based on the PriorBand algorithm. + + Args: + table (pd.DataFrame): The table containing the configurations and their + performance. + rung (int): The current rung of the optimization. + + Returns: + dict[str, Any]: A sampled configuration. + """ + rung_to_fid, rung_sizes = brackets.calculate_sh_rungs( + bounds=self.fid_bounds, + eta=self.eta, + early_stopping_rate=self.early_stopping_rate, + ) + max_rung = max(rung_sizes) + + # Below we will follow the "geometric" spacing + w_random = 1 / (1 + self.eta**rung) + w_prior = 1 - w_random + + completed: pd.DataFrame = table[table["perf"].notna()] # type: ignore + + # To see if we activate incumbent sampling, we check: + # 1) We have at least one fully complete run + # 2) We have spent at least one full SH bracket worth of fidelity + # 3) There is at least one rung with eta evaluations to get the top 1/eta configs + completed_rungs = completed.index.get_level_values("rung") + one_complete_run_at_max_rung = (completed_rungs == max_rung).any() + + # For SH bracket cost, we include the fact we can continue runs, + # i.e. resources for rung 2 discounts the cost of evaluating to rung 1, + # only counting the difference in fidelity cost between rung 2 and rung 1. + cost_per_rung = { + i: rung_to_fid[i] - rung_to_fid.get(i - 1, 0) for i in rung_to_fid + } + + cost_of_one_sh_bracket = sum(rung_sizes[r] * cost_per_rung[r] for r in rung_sizes) + current_cost_used = sum(r * cost_per_rung[r] for r in completed_rungs) + spent_one_sh_bracket_worth_of_fidelity = ( + current_cost_used >= cost_of_one_sh_bracket + ) + + # Check that there is at least rung with `eta` evaluations + rung_counts = completed.groupby("rung").size() + any_rung_with_eta_evals = (rung_counts == self.eta).any() + + # If the conditions are not met, we sample from the prior or randomly depending on + # the geometrically distributed prior and uniform weights + if ( + one_complete_run_at_max_rung is False + or spent_one_sh_bracket_worth_of_fidelity is False + or any_rung_with_eta_evals is False + ): + policy = np.random.choice(["prior", "random"], p=[w_prior, w_random]) + match policy: + case "prior": + return self._sample_prior() + case "random": + return self._sample_random() + case _: + raise RuntimeError(f"Unknown policy: {policy}") + + # Otherwise, we now further split the `prior` weight into `(prior, inc)` + + # 1. Select the top `1//eta` percent of configs at the highest rung supporting it + rungs_with_at_least_eta = rung_counts[rung_counts >= self.eta].index # type: ignore + rung_table: pd.DataFrame = completed[ # type: ignore + completed.index.get_level_values("rung") == rungs_with_at_least_eta.max() + ] + + K = len(rung_table) // self.eta + rung_table.nsmallest(K, columns=["perf"])["config"].tolist() + + # 2. Get the global incumbent + inc_config = completed.loc[completed["perf"].idxmin()]["config"] + + # 3. Calculate a ratio score of how likely each of the top K configs are under + # TODO: [lum]: Here I am simply using fixed values. + # Will maybe have to come up with a way to approximate the pdf for the top + # configs. + inc_ratio = self.inc_ratio + prior_ratio = 1 - self.inc_ratio + + # 4. And finally, we distribute the original w_prior according to this ratio + w_inc = w_prior * inc_ratio + w_prior = w_prior * prior_ratio + + # Normalize to ensure the weights sum to exactly 1.0 + # This handles floating-point precision issues + total_weight = w_prior + w_inc + w_random + w_prior = w_prior / total_weight + w_inc = w_inc / total_weight + w_random = w_random / total_weight + + # Verify weights are valid probabilities (relaxed tolerance for floating-point) + assert np.isclose(w_prior + w_inc + w_random, 1.0, rtol=1e-7, atol=1e-9) + + # Now we use these weights to choose which sampling distribution to sample from + policy = np.random.choice( + ["prior", "inc", "random"], + p=[w_prior, w_inc, w_random], + ) + match policy: + case "prior": + return self._sample_prior() + case "random": + return self._sample_random() + case "inc": + assert inc_config is not None + return self._mutate_inc(inc_config) + raise RuntimeError(f"Unknown policy: {policy}") + + def _sample_prior(self) -> dict[str, Any]: + # TODO: [lum] have a CenterSampler as fallback, not Random + _try_always_priors_sampler = ( + neps.space.neps_spaces.sampling.PriorOrFallbackSampler( + fallback_sampler=neps.space.neps_spaces.sampling.RandomSampler( + predefined_samplings={} + ), + always_use_prior=True, + ) + ) + + _environment_values = {} + _fidelity_attrs = self.space.fidelity_attrs + for fidelity_name, fidelity_obj in _fidelity_attrs.items(): + _environment_values[fidelity_name] = fidelity_obj.upper + + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=_try_always_priors_sampler, + environment_values=_environment_values, + ) + + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) + + def _sample_random(self) -> dict[str, Any]: + _environment_values = {} + _fidelity_attrs = self.space.fidelity_attrs + for fidelity_name, fidelity_obj in _fidelity_attrs.items(): + _environment_values[fidelity_name] = fidelity_obj.upper + + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=neps.space.neps_spaces.sampling.RandomSampler( + predefined_samplings={} + ), + environment_values=_environment_values, + ) + + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) + + def _mutate_inc(self, inc_config: dict[str, Any]) -> dict[str, Any]: + data = neps_space.NepsCompatConverter.from_neps_config(config=inc_config) + + _resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=self.space, + domain_sampler=neps.space.neps_spaces.sampling.MutatateUsingCentersSampler( + predefined_samplings=data.predefined_samplings, + n_mutations=max(1, random.randint(1, int(len(inc_config) / 2))), + ), + environment_values=data.environment_values, + ) + + config = neps_space.NepsCompatConverter.to_neps_config(resolution_context) + return dict(**config) diff --git a/neps/optimizers/neps_random_search.py b/neps/optimizers/neps_random_search.py new file mode 100644 index 000000000..55eb66c32 --- /dev/null +++ b/neps/optimizers/neps_random_search.py @@ -0,0 +1,466 @@ +"""This module implements a simple random search optimizer for a NePS pipeline. +It samples configurations randomly from the pipeline's domain and environment values. +""" + +from __future__ import annotations + +import heapq +import random +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +from neps.optimizers import optimizer +from neps.space.neps_spaces.neps_space import _prepare_sampled_configs, resolve +from neps.space.neps_spaces.parameters import Float, Integer +from neps.space.neps_spaces.sampling import ( + CrossoverByMixingSampler, + CrossoverNotPossibleError, + MutatateUsingCentersSampler, + MutateByForgettingSampler, + PriorOrFallbackSampler, + RandomSampler, +) + +if TYPE_CHECKING: + import neps.state.optimizer as optimizer_state + import neps.state.trial as trial_state + from neps.space.neps_spaces.parameters import PipelineSpace + from neps.state.pipeline_eval import UserResultDict + from neps.state.trial import Trial + + +@dataclass +class NePSRandomSearch: + """A simple random search optimizer for a NePS pipeline. + It samples configurations randomly from the pipeline's domain and environment values. + + Args: + pipeline: The pipeline to optimize, which should be a Pipeline object. + + Raises: + ValueError: If the pipeline is not a Pipeline object. + """ + + def __init__( + self, + pipeline: PipelineSpace, + use_priors: bool = False, # noqa: FBT001, FBT002 + ignore_fidelity: bool | Literal["highest_fidelity"] = False, # noqa: FBT002 + ): + """Initialize the RandomSearch optimizer with a pipeline. + + Args: + pipeline: The pipeline to optimize, which should be a Pipeline object. + + Raises: + ValueError: If the pipeline is not a Pipeline object. + """ + self._pipeline = pipeline + self._random_sampler = RandomSampler(predefined_samplings={}) + self.use_prior = use_priors + self._prior_sampler = PriorOrFallbackSampler( + fallback_sampler=self._random_sampler + ) + self.ignore_fidelity = ignore_fidelity + + def sampled_fidelity_values(self) -> dict[str, Any]: + """Sample fidelity values based on the pipeline's fidelity attributes. + + Returns: + A dictionary mapping fidelity names to their sampled values. + """ + environment_values = {} + fidelity_attrs = self._pipeline.fidelity_attrs + for fidelity_name, fidelity_obj in fidelity_attrs.items(): + if self.ignore_fidelity == "highest_fidelity": + environment_values[fidelity_name] = fidelity_obj.upper + elif not self.ignore_fidelity: + raise ValueError( + "RandomSearch does not support fidelities by default. Consider" + " using a different optimizer or setting `ignore_fidelity=True` or" + " `highest_fidelity`." + ) + # Sample randomly from the fidelity bounds. + elif isinstance(fidelity_obj.domain, Integer): + assert isinstance(fidelity_obj.lower, int) + assert isinstance(fidelity_obj.upper, int) + environment_values[fidelity_name] = random.randint( + fidelity_obj.lower, fidelity_obj.upper + ) + elif isinstance(fidelity_obj.domain, Float): + environment_values[fidelity_name] = random.uniform( + fidelity_obj.lower, fidelity_obj.upper + ) + return environment_values + + def __call__( + self, + trials: Mapping[str, trial_state.Trial], + budget_info: optimizer_state.BudgetInfo | None, + n: int | None = None, + ) -> optimizer.SampledConfig | list[optimizer.SampledConfig]: + """Sample configurations randomly from the pipeline's domain and environment + values. + + Args: + trials: A mapping of trial IDs to Trial objects, representing previous + trials. + budget_info: The budget information for the optimization process. + n: The number of configurations to sample. If None, a single configuration + will be sampled. + + Returns: + A SampledConfig object or a list of SampledConfig objects, depending + on the value of n. + + Raises: + ValueError: If the pipeline is not a Pipeline object or if the trials are + not a valid mapping of trial IDs to Trial objects. + """ + n_prev_trials = len(trials) + n_requested = 1 if n is None else n + return_single = n is None + + if self.use_prior: + chosen_pipelines = [ + resolve( + pipeline=self._pipeline, + domain_sampler=self._prior_sampler, + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested) + ] + else: + chosen_pipelines = [ + resolve( + pipeline=self._pipeline, + domain_sampler=self._random_sampler, + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested) + ] + + return _prepare_sampled_configs(chosen_pipelines, n_prev_trials, return_single) + + def import_trials( + self, + external_evaluations: Sequence[tuple[Mapping[str, Any], UserResultDict]], + trials: Mapping[str, Trial], + ) -> list[optimizer.ImportedConfig]: + """Import external evaluations as trials. + + Args: + external_evaluations: A sequence of tuples containing configuration + dictionaries and their corresponding results. + trials: A mapping of trial IDs to Trial objects, representing previous + trials. + + Returns: + A list of ImportedConfig objects representing the imported trials. + """ + n_trials = len(trials) + imported_configs = [] + for i, (config, result) in enumerate(external_evaluations): + config_id = str(n_trials + i + 1) + imported_configs.append( + optimizer.ImportedConfig( + config=config, + id=config_id, + result=result, + ) + ) + return imported_configs + + +@dataclass +class NePSComplexRandomSearch: + """A complex random search optimizer for a NePS pipeline. + It samples configurations randomly from the pipeline's domain and environment values, + and also performs mutations and crossovers based on previous successful trials. + + Args: + pipeline: The pipeline to optimize, which should be a Pipeline object. + + Raises: + ValueError: If the pipeline is not a Pipeline object. + """ + + def __init__( + self, + pipeline: PipelineSpace, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, # noqa: FBT002 + ): + """Initialize the ComplexRandomSearch optimizer with a pipeline. + + Args: + pipeline: The pipeline to optimize, which should be a Pipeline object. + + Raises: + ValueError: If the pipeline is not a Pipeline object. + """ + self._pipeline = pipeline + + self.ignore_fidelity = ignore_fidelity + + self._random_sampler = RandomSampler( + predefined_samplings={}, + ) + self._try_always_priors_sampler = PriorOrFallbackSampler( + fallback_sampler=self._random_sampler, + always_use_prior=True, + ) + self._sometimes_priors_sampler = PriorOrFallbackSampler( + fallback_sampler=self._random_sampler + ) + self._n_top_trials = 5 + + def sampled_fidelity_values(self) -> dict[str, float | int]: + """Sample fidelity values based on the pipeline's fidelity attributes. + + Returns: + A dictionary mapping fidelity names to their sampled values. + """ + environment_values = {} + fidelity_attrs = self._pipeline.fidelity_attrs + for fidelity_name, fidelity_obj in fidelity_attrs.items(): + if self.ignore_fidelity == "highest_fidelity": + environment_values[fidelity_name] = fidelity_obj.upper + elif not self.ignore_fidelity: + raise ValueError( + "ComplexRandomSearch does not support fidelities by default." + "Consider using a different optimizer or setting" + " `ignore_fidelity=True` or `highest_fidelity`." + ) + # Sample randomly from the fidelity bounds. + elif isinstance(fidelity_obj.domain, Integer): + assert isinstance(fidelity_obj.lower, int) + assert isinstance(fidelity_obj.upper, int) + environment_values[fidelity_name] = random.randint( + fidelity_obj.lower, fidelity_obj.upper + ) + elif isinstance(fidelity_obj.domain, Float): + environment_values[fidelity_name] = random.uniform( + fidelity_obj.lower, fidelity_obj.upper + ) + return environment_values + + def __call__( + self, + trials: Mapping[str, trial_state.Trial], + budget_info: optimizer_state.BudgetInfo | None, + n: int | None = None, + ) -> optimizer.SampledConfig | list[optimizer.SampledConfig]: + """Sample configurations randomly from the pipeline's domain and environment + values, and also perform mutations and crossovers based on previous successful + trials. + + Args: + trials: A mapping of trial IDs to Trial objects, representing previous + trials. + budget_info: The budget information for the optimization process. + n: The number of configurations to sample. If None, a single configuration + will be sampled. + + Returns: + A SampledConfig object or a list of SampledConfig objects, depending + on the value of n. + + Raises: + ValueError: If the pipeline is not a Pipeline object or if the trials are + not a valid mapping of trial IDs to Trial objects. + """ + n_prev_trials = len(trials) + n_requested = 1 if n is None else n + return_single = n is None + + random_pipelines = [ + resolve( + pipeline=self._pipeline, + domain_sampler=self._random_sampler, + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested * 5) + ] + sometimes_priors_pipelines = [ + resolve( + pipeline=self._pipeline, + domain_sampler=self._sometimes_priors_sampler, + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested * 5) + ] + + mutated_incumbents = [] + crossed_over_incumbents = [] + + successful_trials: list[Trial] = list( + filter( + lambda trial: ( + trial.report.reported_as == trial.State.SUCCESS + if trial.report is not None + else False + ), + trials.values(), + ) + ) + if len(successful_trials) > 0: + self._n_top_trials = 5 + top_trials = heapq.nsmallest( + self._n_top_trials, + successful_trials, + key=lambda trial: ( + float(trial.report.objective_to_minimize) + if trial.report + and isinstance(trial.report.objective_to_minimize, float) + else float("inf") + ), + ) # Will have up to `self._n_top_trials` items. + + # Do some mutations. + for top_trial in top_trials: + top_trial_config = top_trial.config + + # Mutate by resampling around some values of the original config. + mutated_incumbents += [ + resolve( + pipeline=self._pipeline, + domain_sampler=MutatateUsingCentersSampler( + predefined_samplings=top_trial_config, + n_mutations=1, + ), + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested * 5) + ] + mutated_incumbents += [ + resolve( + pipeline=self._pipeline, + domain_sampler=MutatateUsingCentersSampler( + predefined_samplings=top_trial_config, + n_mutations=max( + 1, random.randint(1, int(len(top_trial_config) / 2)) + ), + ), + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested * 5) + ] + + # Mutate by completely forgetting some values of the original config. + mutated_incumbents += [ + resolve( + pipeline=self._pipeline, + domain_sampler=MutateByForgettingSampler( + predefined_samplings=top_trial_config, + n_forgets=1, + ), + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested * 5) + ] + mutated_incumbents += [ + resolve( + pipeline=self._pipeline, + domain_sampler=MutateByForgettingSampler( + predefined_samplings=top_trial_config, + n_forgets=max( + 1, random.randint(1, int(len(top_trial_config) / 2)) + ), + ), + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested * 5) + ] + + # Do some crossovers. + if len(top_trials) > 1: + for _ in range(n_requested * 3): + trial_1, trial_2 = random.sample(top_trials, k=2) + + try: + crossover_sampler = CrossoverByMixingSampler( + predefined_samplings_1=trial_1.config, + predefined_samplings_2=trial_2.config, + prefer_first_probability=0.5, + ) + except CrossoverNotPossibleError: + # A crossover was not possible for them. Do nothing. + pass + else: + crossed_over_incumbents.append( + resolve( + pipeline=self._pipeline, + domain_sampler=crossover_sampler, + environment_values=self.sampled_fidelity_values(), + ), + ) + + try: + crossover_sampler = CrossoverByMixingSampler( + predefined_samplings_1=trial_2.config, + predefined_samplings_2=trial_1.config, + prefer_first_probability=0.5, + ) + except CrossoverNotPossibleError: + # A crossover was not possible for them. Do nothing. + pass + else: + crossed_over_incumbents.append( + resolve( + pipeline=self._pipeline, + domain_sampler=crossover_sampler, + environment_values=self.sampled_fidelity_values(), + ), + ) + + all_sampled_pipelines = [ + *random_pipelines, + *sometimes_priors_pipelines, + *mutated_incumbents, + *crossed_over_incumbents, + ] + + # Here we can have a model which picks from all the sampled pipelines. + # Currently, we just pick randomly from them. + chosen_pipelines = random.sample(all_sampled_pipelines, k=n_requested) + + if n_prev_trials == 0: + # In this case, always include the prior pipeline. + prior_pipeline = resolve( + pipeline=self._pipeline, + domain_sampler=self._try_always_priors_sampler, + environment_values=self.sampled_fidelity_values(), + ) + chosen_pipelines[0] = prior_pipeline + + return _prepare_sampled_configs(chosen_pipelines, n_prev_trials, return_single) + + def import_trials( + self, + external_evaluations: Sequence[tuple[Mapping[str, Any], UserResultDict]], + trials: Mapping[str, Trial], + ) -> list[optimizer.ImportedConfig]: + """Import external evaluations as trials. + + Args: + external_evaluations: A sequence of tuples containing configuration + dictionaries and their corresponding results. + trials: A mapping of trial IDs to Trial objects, representing previous + trials. + + Returns: + A list of ImportedConfig objects representing the imported trials. + """ + n_trials = len(trials) + imported_configs = [] + for i, (config, result) in enumerate(external_evaluations): + config_id = str(n_trials + i + 1) + imported_configs.append( + optimizer.ImportedConfig( + config=config, + id=config_id, + result=result, + ) + ) + return imported_configs diff --git a/neps/optimizers/neps_regularized_evolution.py b/neps/optimizers/neps_regularized_evolution.py new file mode 100644 index 000000000..62ea39bf4 --- /dev/null +++ b/neps/optimizers/neps_regularized_evolution.py @@ -0,0 +1,416 @@ +"""This module implements a Regularized Evolution optimizer for NEPS.""" + +from __future__ import annotations + +import heapq +import random +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +from neps.optimizers.optimizer import ImportedConfig +from neps.space.neps_spaces.neps_space import ( + SamplingResolutionContext, + _prepare_sampled_configs, + resolve, +) +from neps.space.neps_spaces.parameters import Float, Integer, PipelineSpace +from neps.space.neps_spaces.sampling import ( + CrossoverByMixingSampler, + CrossoverNotPossibleError, + MutatateUsingCentersSampler, + MutateByForgettingSampler, + PriorOrFallbackSampler, + RandomSampler, +) + +if TYPE_CHECKING: + import neps.state.optimizer as optimizer_state + import neps.state.trial as trial_state + from neps.optimizers import optimizer + from neps.state.pipeline_eval import UserResultDict + from neps.state.trial import Trial + + +@dataclass +class NePSRegularizedEvolution: + """A Regularized Evolution optimizer for a NePS pipeline. + It samples configurations based on mutations and crossovers of previous successful + trials, using a tournament selection mechanism. + + Args: + pipeline: The pipeline to optimize. + population_size: The size of the population for evolution. + tournament_size: The size of the tournament for selecting parents. + use_priors: Whether to use priors when sampling if available. + mutation_type: The type of mutation to use (e.g., "mutate_best", + "crossover_top_2"). If a float is provided, it is interpreted as the + probability of choosing mutation, compared to the probability of crossover. + n_mutations: The number of mutations to apply. A fixed integer, "random" for + a random number between 1 and half the parameters, or "half" to mutate + half the parameters. + n_forgets: The number of parameters to forget. A fixed integer, "random" for + a random number between 1 and half the parameters, or "half" to forget + half the parameters. + ignore_fidelity: Whether to ignore_fidelity when sampling. If set to "highest + fidelity", the highest fidelity values will be used. If True, fidelity + values will be sampled randomly. + """ + + def __init__( + self, + pipeline: PipelineSpace, + population_size: int = 20, + tournament_size: int = 5, + use_priors: bool = True, # noqa: FBT001, FBT002 + mutation_type: float | Literal["mutate_best", "crossover_top_2"] = 0.5, + n_mutations: int | Literal["random", "half"] | None = "random", + n_forgets: int | Literal["random", "half"] | None = None, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, # noqa: FBT002 + ): + """Initialize the RegularizedEvolution optimizer with a pipeline. + + Args: + pipeline: The pipeline to optimize, which should be a Pipeline object. + population_size: The size of the population for evolution. + tournament_size: The size of the tournament for selecting parents. + use_priors: Whether to use priors when sampling if available. + mutation_type: The type of mutation to use (e.g., "mutate_best", + "crossover_top_2"). If a float is provided, it is interpreted as the + probability of choosing mutation, compared to the probability of + crossover. + n_mutations: The number of mutations to apply. A fixed integer, "random" for + a random number between 1 and half the parameters, or "half" to mutate + half the parameters. + n_forgets: The number of parameters to forget. A fixed integer, "random" for + a random number between 1 and half the parameters, or "half" to forget + half the parameters. + ignore_fidelity: Whether to ignore_fidelity when sampling. If set to "highest + fidelity", the highest fidelity values will be used. If True, fidelity + values will be sampled randomly. + + Raises: + ValueError: If the pipeline is not a Pipeline object. + """ + self._pipeline = pipeline + + self._random_or_prior_sampler: RandomSampler | PriorOrFallbackSampler = ( + RandomSampler( + predefined_samplings={}, + ) + ) + if use_priors: + self._random_or_prior_sampler = PriorOrFallbackSampler( + fallback_sampler=self._random_or_prior_sampler + ) + assert population_size >= tournament_size, ( + "Population size must be greater than or equal to tournament size." + ) + self._tournament_size = tournament_size + self._population_size = population_size + self._mutation_type = mutation_type + self._n_mutations = n_mutations + self._n_forgets = n_forgets + self._ignore_fidelity = ignore_fidelity + + def _mutate_best( + self, top_trial_config: Mapping[str, Any] + ) -> tuple[PipelineSpace, SamplingResolutionContext]: + """Mutate the best trial's config by resampling or forgetting parameters. + + Args: + top_trial_config: The configuration of the best trial to mutate. + + Returns: + A mutated configuration (PipelineSpace and context tuple). + + Raises: + ValueError: If both n_mutations and n_forgets are None. + """ + if self._n_mutations: + n_mut = ( + self._n_mutations + if isinstance(self._n_mutations, int) + else ( + random.randint(1, len(top_trial_config) // 2) + if self._n_mutations == "random" + else len(top_trial_config) // 2 + ) + ) + return resolve( + pipeline=self._pipeline, + domain_sampler=MutatateUsingCentersSampler( + predefined_samplings=top_trial_config, + n_mutations=n_mut, + ), + environment_values=self.sampled_fidelity_values(), + ) + if self._n_forgets: + n_forg = ( + self._n_forgets + if isinstance(self._n_forgets, int) + else ( + random.randint(1, len(top_trial_config) // 2) + if self._n_forgets == "random" + else max(1, len(top_trial_config) // 2) + ) + ) + return resolve( + pipeline=self._pipeline, + domain_sampler=MutateByForgettingSampler( + predefined_samplings=top_trial_config, + n_forgets=n_forg, + ), + environment_values=self.sampled_fidelity_values(), + ) + raise ValueError("At least one of n_mutations or n_forgets must not be None.") + + def _crossover_top_2( + self, sorted_trials: list[Trial] + ) -> tuple[PipelineSpace, SamplingResolutionContext]: + """Perform crossover between top trials from the tournament. + + Args: + sorted_trials: List of configurations sorted by objective (best first). + + Returns: + A configuration created by crossover (PipelineSpace and context tuple), + or a mutated config if crossover fails. + """ + # Create all possible crossovers between the top trials, sorted by smallest + # combined index. + all_crossovers = [ + (x, y) + for x in range(len(sorted_trials)) + for y in range(len(sorted_trials)) + if x < y + ] + all_crossovers.sort(key=lambda pair: pair[0] + pair[1]) + + for n, (config_1, config_2) in enumerate(all_crossovers): + top_trial_config = sorted_trials[config_1].config + second_best_trial_config = sorted_trials[config_2].config + + # Crossover between the best two trials' configs to create a new config. + try: + crossover_sampler = CrossoverByMixingSampler( + predefined_samplings_1=top_trial_config, + predefined_samplings_2=second_best_trial_config, + prefer_first_probability=0.5, + ) + except CrossoverNotPossibleError: + # A crossover was not possible for them. Increase configs and try again. + # If we have tried all crossovers, mutate the best instead. + if n == len(all_crossovers) - 1: + # Mutate 50% of the top trial's config. + return resolve( + pipeline=self._pipeline, + domain_sampler=MutatateUsingCentersSampler( + predefined_samplings=top_trial_config, + n_mutations=max(1, int(len(top_trial_config) / 2)), + ), + environment_values=self.sampled_fidelity_values(), + ) + continue + else: + return resolve( + pipeline=self._pipeline, + domain_sampler=crossover_sampler, + environment_values=self.sampled_fidelity_values(), + ) + + # Fallback in case all crossovers fail (shouldn't happen, but be safe) + return self._mutate_best(sorted_trials[0].config) + + def sampled_fidelity_values( + self, + ) -> Mapping[str, float | int]: + """Get the sampled fidelity values used in the optimizer. + + Returns: + A mapping of fidelity names to their sampled values. + """ + + environment_values = {} + fidelity_attrs = self._pipeline.fidelity_attrs + for fidelity_name, fidelity_obj in fidelity_attrs.items(): + # If the user specifically asked for the highest fidelity, use that. + if self._ignore_fidelity == "highest_fidelity": + environment_values[fidelity_name] = fidelity_obj.upper + # If the user asked to ignore fidelities, sample a value randomly from the + # domain. + elif self._ignore_fidelity is True: + # Sample randomly from the fidelity bounds. + if isinstance(fidelity_obj.domain, Integer): + assert isinstance(fidelity_obj.lower, int) + assert isinstance(fidelity_obj.upper, int) + environment_values[fidelity_name] = random.randint( + fidelity_obj.lower, fidelity_obj.upper + ) + elif isinstance(fidelity_obj.domain, Float): + environment_values[fidelity_name] = random.uniform( + fidelity_obj.lower, fidelity_obj.upper + ) + # By default we don't support fidelities unless explicitly requested. + else: + raise ValueError( + "RegularizedEvolution does not support fidelities by default. " + "Consider using a different optimizer or setting " + "`ignore_fidelity=True` or `highest_fidelity`." + ) + return environment_values + + def __call__( + self, + trials: Mapping[str, trial_state.Trial], + budget_info: optimizer_state.BudgetInfo | None, + n: int | None = None, + ) -> optimizer.SampledConfig | list[optimizer.SampledConfig]: + """Sample configurations randomly from the pipeline's domain and environment + values, and also perform mutations and crossovers based on previous successful + trials. + + Args: + trials: A mapping of trial IDs to Trial objects, representing previous + trials. + budget_info: The budget information for the optimization process. + n: The number of configurations to sample. If None, a single configuration + will be sampled. + + Returns: + A SampledConfig object or a list of SampledConfig objects, depending + on the value of n. + + Raises: + ValueError: If the pipeline is not a Pipeline object or if the trials are + not a valid mapping of trial IDs to Trial objects. + """ + n_prev_trials = len(trials) + n_requested = 1 if n is None else n + + if n_prev_trials < self._population_size: + # Just do random sampling until we have enough trials. + random_pipelines = [ + resolve( + pipeline=self._pipeline, + domain_sampler=self._random_or_prior_sampler, + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested) + ] + + return _prepare_sampled_configs( + random_pipelines, n_prev_trials, n_requested == 1 + ) + + successful_trials: list[Trial] = list( + filter( + lambda trial: ( + trial.report.reported_as == trial.State.SUCCESS + if trial.report is not None + else False + ), + trials.values(), + ) + ) + + # If we have no successful trials yet, fall back to random sampling. + if len(successful_trials) == 0: + random_pipelines = [ + resolve( + pipeline=self._pipeline, + domain_sampler=self._random_or_prior_sampler, + environment_values=self.sampled_fidelity_values(), + ) + for _ in range(n_requested) + ] + + return _prepare_sampled_configs( + random_pipelines, n_prev_trials, n_requested == 1 + ) + + return_pipelines = [] + + for _ in range(n_requested): + # Select the most recent trials to form the tournament. + # We want the last (most recent) self._population_size successful trials. + latest_trials = heapq.nlargest( + self._population_size, + successful_trials, + key=lambda trial: ( + trial.metadata.time_end + if trial.metadata and isinstance(trial.metadata.time_end, float) + else 0.0 + ), + ) + + tournament_trials = [ + random.sample((latest_trials), k=1)[0] + for _ in range(min(self._tournament_size, len(latest_trials))) + ] + + # Sort the tournament by objective and pick the best as the parent. + def _obj_key(trial: Trial) -> float: + return ( + float(trial.report.objective_to_minimize) + if trial.report + and isinstance(trial.report.objective_to_minimize, float) + else float("inf") + ) + + sorted_trials = sorted(tournament_trials, key=_obj_key) + + top_trial_config = sorted_trials[0].config + + # Mutate or crossover the best trial's config to create a new config. + if self._mutation_type == "mutate_best": + mutated_incumbent = self._mutate_best(top_trial_config) + return_pipelines.append(mutated_incumbent) + elif self._mutation_type == "crossover_top_2": + crossed_over_incumbent = self._crossover_top_2(sorted_trials) + return_pipelines.append(crossed_over_incumbent) + elif isinstance(self._mutation_type, float): + if self._mutation_type < 0.0 or self._mutation_type > 1.0: + raise ValueError( + f"Invalid mutation probability: {self._mutation_type}. " + "It must be between 0.0 and 1.0." + ) + rand_val = random.random() + + if rand_val < self._mutation_type: + return_pipelines.append(self._mutate_best(top_trial_config)) + else: + return_pipelines.append(self._crossover_top_2(sorted_trials)) + else: + raise ValueError(f"Invalid mutation type: {self._mutation_type}") + + return _prepare_sampled_configs(return_pipelines, n_prev_trials, n_requested == 1) + + def import_trials( + self, + external_evaluations: Sequence[tuple[Mapping[str, Any], UserResultDict]], + trials: Mapping[str, Trial], + ) -> list[ImportedConfig]: + """Import external evaluations as trials. + + Args: + external_evaluations: A sequence of tuples containing configurations and + their evaluation results. + trials: A mapping of trial IDs to Trial objects, representing existing + trials. + + Returns: + A list of ImportedConfig objects representing the imported trials. + """ + n_trials = len(trials) + imported_configs = [] + for i, (config, result) in enumerate(external_evaluations): + config_id = str(n_trials + i + 1) + imported_configs.append( + ImportedConfig( + config=config, + id=config_id, + result=result, + ) + ) + return imported_configs diff --git a/neps/optimizers/priorband.py b/neps/optimizers/priorband.py index 9d6d23e4b..c12358ccf 100644 --- a/neps/optimizers/priorband.py +++ b/neps/optimizers/priorband.py @@ -47,12 +47,12 @@ class PriorBandSampler: fid_bounds: tuple[int, int] | tuple[float, float] """The fidelity bounds.""" - def sample_config(self, table: pd.DataFrame, rung: int) -> dict[str, Any]: + def sample_config(self, table: pd.DataFrame, rung: int) -> dict[str, Any]: # noqa: PLR0915 """Samples a configuration using the PriorBand algorithm. Args: table: The table of all the trials that have been run. - rung_to_sample_for: The rung to sample for. + rung: The rung to sample for. Returns: The sampled configuration. @@ -145,7 +145,16 @@ def sample_config(self, table: pd.DataFrame, rung: int) -> dict[str, Any]: # 4. And finally, we distribute the original w_prior according to this ratio w_inc = w_prior * inc_ratio w_prior = w_prior * prior_ratio - assert np.isclose(w_prior + w_inc + w_random, 1.0) + + # Normalize to ensure the weights sum to exactly 1.0 + # This handles floating-point precision issues + total_weight = w_prior + w_inc + w_random + w_prior = w_prior / total_weight + w_inc = w_inc / total_weight + w_random = w_random / total_weight + + # Verify weights are valid probabilities (relaxed tolerance for floating-point) + assert np.isclose(w_prior + w_inc + w_random, 1.0, rtol=1e-7, atol=1e-9) # Now we use these weights to choose which sampling distribution to sample from policy = np.random.choice( diff --git a/neps/optimizers/random_search.py b/neps/optimizers/random_search.py index 0de6d28a3..e1b484b37 100644 --- a/neps/optimizers/random_search.py +++ b/neps/optimizers/random_search.py @@ -5,6 +5,8 @@ from typing import TYPE_CHECKING, Any from neps.optimizers.optimizer import ImportedConfig, SampledConfig +from neps.space.neps_spaces.neps_space import convert_neps_to_classic_search_space +from neps.space.neps_spaces.parameters import PipelineSpace if TYPE_CHECKING: from neps.sampling import Sampler @@ -17,7 +19,7 @@ class RandomSearch: """A simple random search optimizer.""" - space: SearchSpace + space: SearchSpace | PipelineSpace encoder: ConfigEncoder sampler: Sampler @@ -27,6 +29,15 @@ def __call__( budget_info: BudgetInfo | None, n: int | None = None, ) -> SampledConfig | list[SampledConfig]: + if isinstance(self.space, PipelineSpace): + converted_space = convert_neps_to_classic_search_space(self.space) + if converted_space is not None: + self.space = converted_space + else: + raise ValueError( + "This optimizer only supports HPO search spaces, please use a NePS" + " space-compatible optimizer." + ) n_trials = len(trials) _n = 1 if n is None else n configs = self.sampler.sample(_n, to=self.encoder.domains) diff --git a/neps/optimizers/utils/brackets.py b/neps/optimizers/utils/brackets.py index 690c5fab2..8150f7035 100644 --- a/neps/optimizers/utils/brackets.py +++ b/neps/optimizers/utils/brackets.py @@ -10,7 +10,6 @@ if TYPE_CHECKING: import pandas as pd - from pandas import Index logger = logging.getLogger(__name__) @@ -337,41 +336,39 @@ def create_repeating( Brackets which have each subselected the table with the corresponding rung sizes. """ - # Split the trials by their unique_id, taking batches of K at a time, which will - # gives us N = len(unique_is) / K brackets in total. - # - # Here, unique_id referes to the `1` in config_1_0 i.e. id = 1, rung = 0 - # - # 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 ... - # | bracket1 | bracket 2 | ... | - - # K is the number of configurations in the lowest rung, which is how many unique - # ids are needed to fill a single bracket. - K = rung_sizes[min(rung_sizes)] - - # N is the number of brackets we need to create to accomodate all the unique ids. - # First we need all of the unique ids. - uniq_ids = table.index.get_level_values("id").unique() - - # The formula (len(uniq_ids) + K) // K is used instead of - # len(uniq_ids) // K. reason: make to ensure that even if the number of - # unique IDs is less than K, at least one bracket is created - N = (len(uniq_ids) + K) // K - - # Now we take the unique ids and split them into batches of size K - bracket_id_slices: list[Index] = [uniq_ids[i * K : (i + 1) * K] for i in range(N)] - - # And now select the data for each of the unique_ids in the bracket - bracket_datas = [ - table.loc[bracket_unique_ids] for bracket_unique_ids in bracket_id_slices - ] - - # This will give us a list of dictionaries, where each element `n` of the - # list is on of the `N` brackets, and the dictionary at element `n` maps - # from a rung, to the slice of the data for that rung. - all_N_bracket_datas = [ - dict(iter(d.groupby(level="rung", sort=False))) for d in bracket_datas - ] + # Group configs by their starting rung (minimum rung where they first appear). + # This ensures each bracket's bottom rung fills with configs that actually + # evaluate there, which is critical for proper successive halving. + bottom_rung = min(rung_sizes) + bottom_capacity = rung_sizes[bottom_rung] + + # Group all configs by their starting rung + configs_by_starting_rung: dict[int, list[int]] = {} + for config_id in table.index.get_level_values("id").unique(): + config_rungs = table.loc[config_id].index.get_level_values("rung") + starting_rung = config_rungs.min() + if starting_rung not in configs_by_starting_rung: + configs_by_starting_rung[starting_rung] = [] + configs_by_starting_rung[starting_rung].append(config_id) + + # Create brackets by taking configs that start at the bottom rung + brackets_data = [] + while configs_by_starting_rung.get(bottom_rung, []): + # Take up to bottom_capacity configs that start at bottom rung + available_ids = configs_by_starting_rung[bottom_rung] + bracket_ids = available_ids[:bottom_capacity] + configs_by_starting_rung[bottom_rung] = available_ids[bottom_capacity:] + + if not configs_by_starting_rung[bottom_rung]: + del configs_by_starting_rung[bottom_rung] + + # Create data for this bracket + bracket_table = table.loc[bracket_ids] + bracket_data = dict(iter(bracket_table.groupby(level="rung", sort=False))) + brackets_data.append(bracket_data) + + # Always add one empty bracket for new samples + brackets_data.append({}) # Used if there is nothing for one of the rungs empty_slice = table.loc[[]] @@ -385,7 +382,7 @@ def create_repeating( is_multi_objective=is_multi_objective, mo_selector=mo_selector, ) - for bracket_data in all_N_bracket_datas + for bracket_data in brackets_data ] @@ -538,53 +535,43 @@ def create_repeating( HyperbandBrackets which have each subselected the table with the corresponding rung sizes. """ - all_ids = table.index.get_level_values("id").unique() + # Group configs by their starting rung (minimum rung where they first appear) + # This ensures warmstarts at different fidelities are properly distributed + configs_by_starting_rung: dict[int, list[int]] = {} + for config_id in table.index.get_level_values("id").unique(): + config_rungs = table.loc[config_id].index.get_level_values("rung") + starting_rung = config_rungs.min() + if starting_rung not in configs_by_starting_rung: + configs_by_starting_rung[starting_rung] = [] + configs_by_starting_rung[starting_rung].append(config_id) - # Split the ids into N hyperband brackets of size K. - # K is sum of number of configurations in the lowest rung of each SH bracket - # - # For example: - # > bracket_layouts = [ - # > {0: 81, 1: 27, 2: 9, 3: 3, 4: 1}, - # > {1: 27, 2: 9, 3: 3, 4: 1}, - # > {2: 9, 3: 3, 4: 1}, - # > ... - # > ] - # - # Corresponds to: - # bracket1 - [rung_0: 81, rung_1: 27, rung_2: 9, rung_3: 3, rung_4: 1] - # bracket2 - [rung_1: 27, rung_2: 9, rung_3: 3, rung_4: 1] - # bracket3 - [rung_2: 9, rung_3: 3, rung_4: 1] - # ... - # > K = 81 + 27 + 9 + ... - # - bottom_rung_sizes = [sh[min(sh.keys())] for sh in bracket_layouts] - K = sum(bottom_rung_sizes) - N = max(len(all_ids) // K + 1, 1) + empty_slice = table.loc[[]] + hb_brackets: list[list[Sync]] = [] - hb_id_slices: list[Index] = [all_ids[i * K : (i + 1) * K] for i in range(N)] + # Assign configs to brackets based on which rung they start at + # Keep creating Hyperband cycles until all configs are allocated + while any(configs_by_starting_rung.values()): + sh_brackets: list[Sync] = [] - # Used if there is nothing for one of the rungs - empty_slice = table.loc[[]] + for layout in bracket_layouts: + bottom_rung = min(layout.keys()) + capacity_at_bottom = layout[bottom_rung] - # Now for each of our HB brackets, we need to split them into the SH brackets - hb_brackets: list[list[Sync]] = [] + # Take configs that start at this bracket's bottom rung + available_ids = configs_by_starting_rung.get(bottom_rung, []) + bracket_ids = available_ids[:capacity_at_bottom] - offsets = np.cumsum([0, *bottom_rung_sizes]) - for hb_ids in hb_id_slices: - # Split the ids into each of the respective brackets, e.g. [81, 27, 9, ...] - ids_for_each_bracket = [hb_ids[s:e] for s, e in pairwise(offsets)] + # Remove assigned configs from pool + if bottom_rung in configs_by_starting_rung: + configs_by_starting_rung[bottom_rung] = available_ids[ + capacity_at_bottom: + ] + if not configs_by_starting_rung[bottom_rung]: + del configs_by_starting_rung[bottom_rung] - # Select the data for each of the configs allocated to these sh_brackets - data_for_each_bracket = [table.loc[_ids] for _ids in ids_for_each_bracket] + # Get data for assigned configs across all their rungs + data_for_bracket = table.loc[bracket_ids] if bracket_ids else empty_slice - # Create the bracket - sh_brackets: list[Sync] = [] - for data_for_bracket, layout in zip( - data_for_each_bracket, - bracket_layouts, - strict=True, - ): rung_data = dict(iter(data_for_bracket.groupby(level="rung", sort=False))) bracket = Sync( rungs=[ @@ -602,6 +589,24 @@ def create_repeating( hb_brackets.append(sh_brackets) + # Always add one empty Hyperband cycle for new samples + sh_brackets = [] + for layout in bracket_layouts: + bracket = Sync( + rungs=[ + Rung( + value=rung, + capacity=capacity, + table=empty_slice, + ) + for rung, capacity in layout.items() + ], + is_multi_objective=is_multi_objective, + mo_selector=mo_selector, + ) + sh_brackets.append(bracket) + hb_brackets.append(sh_brackets) + return [cls(sh_brackets=sh_brackets) for sh_brackets in hb_brackets] def next(self) -> BracketAction: diff --git a/neps/optimizers/utils/grid.py b/neps/optimizers/utils/grid.py index 720dd7713..3fb5991c1 100644 --- a/neps/optimizers/utils/grid.py +++ b/neps/optimizers/utils/grid.py @@ -1,28 +1,38 @@ from __future__ import annotations from itertools import product -from typing import Any +from typing import Any, Literal import torch -from neps.space import Categorical, Constant, Domain, Float, Integer, SearchSpace +from neps import Categorical, Fidelity, Float, Integer, PipelineSpace +from neps.space import ( + Domain, + HPOCategorical, + HPOConstant, + HPOFloat, + HPOInteger, + SearchSpace, +) +from neps.space.neps_spaces import neps_space +from neps.space.neps_spaces.sampling import RandomSampler -def make_grid( - space: SearchSpace, +def make_grid( # noqa: PLR0912, PLR0915, C901 + space: SearchSpace | PipelineSpace, *, size_per_numerical_hp: int = 10, - ignore_fidelity: bool = True, + ignore_fidelity: bool | Literal["highest_fidelity"] = False, ) -> list[dict[str, Any]]: """Get a grid of configurations from the search space. - For [`Float`][neps.space.Float] and [`Integer`][neps.space.Integer] + For [`Float`][neps.space.HPOFloat] and [`Integer`][neps.space.HPOInteger] the parameter `size_per_numerical_hp=` is used to determine a grid. - For [`Categorical`][neps.space.Categorical] + For [`Categorical`][neps.space.HPOCategorical] hyperparameters, we include all the choices in the grid. - For [`Constant`][neps.space.Constant] hyperparameters, + For [`Constant`][neps.space.HPOConstant] hyperparameters, we include the constant value in the grid. Args: @@ -32,29 +42,102 @@ def make_grid( A list of configurations from the search space. """ param_ranges: dict[str, list[Any]] = {} - for name, hp in space.items(): - match hp: - case Categorical(): - param_ranges[name] = list(hp.choices) - case Constant(): - param_ranges[name] = [hp.value] - case Integer() | Float(): - if hp.is_fidelity and ignore_fidelity: - param_ranges[name] = [hp.upper] - continue + if isinstance(space, SearchSpace): + for name, hp in space.items(): + match hp: + case HPOCategorical(): + param_ranges[name] = list(hp.choices) + case HPOConstant(): + param_ranges[name] = [hp.value] + case HPOInteger() | HPOFloat(): + if hp.is_fidelity: + match ignore_fidelity: + case "highest_fidelity": + param_ranges[name] = [hp.upper] + continue + case True: + param_ranges[name] = [hp.lower, hp.upper] + case False: + raise ValueError( + "Grid search does not support fidelity " + "natively. Please use the" + "ignore_fidelity parameter." + ) + if hp.domain.cardinality is None: + steps = size_per_numerical_hp + else: + steps = min(size_per_numerical_hp, hp.domain.cardinality) - if hp.domain.cardinality is None: - steps = size_per_numerical_hp + xs = torch.linspace(0, 1, steps=steps) + numeric_values = hp.domain.cast(xs, frm=Domain.unit_float()) + uniq_values = torch.unique(numeric_values).tolist() + param_ranges[name] = uniq_values + case _: + raise NotImplementedError(f"Unknown Parameter type: {type(hp)}\n{hp}") + keys = list(space.keys()) + values = product(*param_ranges.values()) + return [dict(zip(keys, p, strict=False)) for p in values] + if isinstance(space, PipelineSpace): + fid_ranges: dict[str, list[float]] = {} + for name, hp in space.get_attrs().items(): + if isinstance(hp, Categorical): + if isinstance(hp.choices, tuple): # type: ignore[unreachable] + param_ranges[name] = list(range(len(hp.choices))) else: - steps = min(size_per_numerical_hp, hp.domain.cardinality) - + raise NotImplementedError( + "Grid search only supports categorical choices as tuples." + ) + elif isinstance(hp, Fidelity): + if ignore_fidelity == "highest_fidelity": # type: ignore[unreachable] + fid_ranges[name] = [hp.upper] + continue + if ignore_fidelity is True: + fid_ranges[name] = [hp.lower, hp.upper] + continue + raise ValueError( + "Grid search does not support fidelity natively." + " Please use the ignore_fidelity parameter." + ) + elif isinstance(hp, Integer | Float): + steps = size_per_numerical_hp # type: ignore[unreachable] xs = torch.linspace(0, 1, steps=steps) - numeric_values = hp.domain.cast(xs, frm=Domain.unit_float()) + numeric_values = xs * (hp.upper - hp.lower) + hp.lower + if isinstance(hp, Integer): + numeric_values = torch.round(numeric_values) uniq_values = torch.unique(numeric_values).tolist() param_ranges[name] = uniq_values - case _: - raise NotImplementedError(f"Unknown Parameter type: {type(hp)}\n{hp}") - values = product(*param_ranges.values()) - keys = list(space.keys()) + else: + raise NotImplementedError( + f"Parameter type: {type(hp)}\n{hp} not supported yet in GridSearch" + ) + keys = list(param_ranges.keys()) + values = product(*param_ranges.values()) + config_dicts = [dict(zip(keys, p, strict=False)) for p in values] + keys_fid = list(fid_ranges.keys()) + values_fid = product(*fid_ranges.values()) + fid_dicts = [dict(zip(keys_fid, p, strict=False)) for p in values_fid] + configs = [] + random_config = neps_space.NepsCompatConverter.to_neps_config( + neps_space.resolve( + pipeline=space, + domain_sampler=RandomSampler(predefined_samplings={}), + environment_values=fid_dicts[0], + )[1] + ) + + for config_dict in config_dicts: + for fid_dict in fid_dicts: + new_config = {} + for param in random_config: + for key in config_dict: + if key in param: + new_config[param] = config_dict[key] + for key in fid_dict: + if key in param: + new_config[param] = fid_dict[key] + configs.append(new_config) + return configs - return [dict(zip(keys, p, strict=False)) for p in values] + raise TypeError( + f"Unsupported space type: {type(space)}" + ) # More informative than None diff --git a/neps/optimizers/utils/initial_design.py b/neps/optimizers/utils/initial_design.py index 615a5a257..81a895489 100644 --- a/neps/optimizers/utils/initial_design.py +++ b/neps/optimizers/utils/initial_design.py @@ -24,7 +24,6 @@ def make_initial_design( """Generate the initial design of the optimization process. Args: - space: The search space to use. encoder: The encoder to use for encoding/decoding configurations. sampler: The sampler to use for the initial design. diff --git a/neps/runtime.py b/neps/runtime.py index 3e6ff807c..10a9e7195 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -12,9 +12,9 @@ import shutil import time import traceback -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import asdict, dataclass from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Literal @@ -35,6 +35,7 @@ WorkerFailedToGetPendingTrialsError, WorkerRaiseError, ) +from neps.space.neps_spaces.neps_space import NepsCompatConverter, PipelineSpace from neps.state import ( BudgetInfo, DefaultReportValues, @@ -48,10 +49,16 @@ WorkerSettings, evaluate_trial, ) -from neps.status.status import _initiate_summary_csv, status +from neps.status.status import ( + _build_incumbent_content, + _build_optimal_set_content, + _initiate_summary_csv, + status, +) from neps.utils.common import gc_disabled if TYPE_CHECKING: + from neps import SearchSpace from neps.optimizers import OptimizerInfo from neps.optimizers.optimizer import AskFunction @@ -157,6 +164,33 @@ def _set_global_trial(trial: Trial) -> Iterator[None]: _CURRENTLY_RUNNING_TRIAL_IN_PROCESS = None +@dataclass +class ResourceUsage: + """Container for tracking cumulative resource usage.""" + + evaluations: int = 0 + cost: float = 0.0 + fidelities: float = 0.0 + time: float = 0.0 + + def __iadd__(self, other: ResourceUsage) -> ResourceUsage: + """Allows syntax: usage += other_usage.""" + self.evaluations += other.evaluations + self.cost += other.cost + self.fidelities += other.fidelities + self.time += other.time + return self + + def to_trajectory_dict(self) -> dict[str, float | int]: + """Converts usage to the dictionary keys expected by the trajectory file.""" + return { + "cumulative_evaluations": self.evaluations, + "cumulative_cost": self.cost, + "cumulative_fidelities": self.fidelities, + "cumulative_time": self.time, + } + + # NOTE: This class is quite stateful and has been split up quite a bit to make testing # interleaving of workers easier. This comes at the cost of more fragmented code. @dataclass @@ -282,85 +316,201 @@ def _check_shared_error_stopping_criterion(self) -> str | Literal[False]: return False - def _check_global_stopping_criterion( + def _calculate_total_resource_usage( # noqa: C901 self, trials: Mapping[str, Trial], - ) -> str | Literal[False]: - # worker related stopping criterion - worker_trials = { - _id: trial - for _id, trial in trials.items() - if trial.metadata.evaluating_worker_id == self.worker_id - } - if self.settings.evaluations_to_spend is not None: - if self.settings.include_in_progress_evaluations_towards_maximum: - count = sum( - 1 - for _, trial in worker_trials.items() - if trial.metadata.state != Trial.State.PENDING - ) - else: - # This indicates they have completed. - count = sum( - 1 for _, trial in worker_trials.items() if trial.report is not None + subset_worker_id: str | None = None, + *, + include_in_progress: bool = False, + ) -> ResourceUsage: + """Calculates total resources returning a typed usage object. + + Args: + trials: Dictionary of trials to calculate from. + subset_worker_id: If provided, only calculates for + trials evaluated by this worker ID. + include_in_progress: Whether to include incomplete trials. + """ + relevant_trials = list(trials.values()) + if subset_worker_id is not None: + relevant_trials = [ + t + for t in relevant_trials + if t.metadata.evaluating_worker_id == subset_worker_id + ] + + fidelity_name = None + if hasattr(self.optimizer, "space"): + if isinstance(self.optimizer.space, PipelineSpace): + if self.optimizer.space.fidelity_attrs: + fidelity_name = next(iter(self.optimizer.space.fidelity_attrs.keys())) + fidelity_name = ( + f"{NepsCompatConverter._ENVIRONMENT_PREFIX}{fidelity_name}" + ) + elif self.optimizer.space.fidelities: + fidelity_name = next(iter(self.optimizer.space.fidelities.keys())) + + usage = ResourceUsage() + + for trial in relevant_trials: + if not ( + trial.report is not None + or ( + include_in_progress and trial.metadata.state == Trial.State.EVALUATING ) + ): + continue + usage.evaluations += 1 + if trial.report and trial.report.cost is not None: + usage.cost += trial.report.cost + + # Handle time: either from report or calculate from metadata + if trial.report and trial.report.evaluation_duration is not None: + usage.time += trial.report.evaluation_duration + elif ( + trial.metadata.time_started is not None + and trial.metadata.time_end is not None + ): + usage.time += trial.metadata.time_end - trial.metadata.time_started + + if ( + fidelity_name + and fidelity_name in trial.config + and trial.config[fidelity_name] is not None + ): + usage.fidelities += trial.config[fidelity_name] + + return usage + + def _check_global_stopping_criterion( # noqa: C901 + self, + trials: Mapping[str, Trial], + log_status: bool = False, # noqa: FBT001, FBT002 + ) -> tuple[str | Literal[False], ResourceUsage]: + """Evaluates if any global stopping criterion has been met. - if count >= self.settings.evaluations_to_spend: - return ( - "Worker has reached the maximum number of evaluations it is allowed" - f" to do as given by `{self.settings.evaluations_to_spend=}`." - "\nTo allow more evaluations, increase this value or use a different" - " stopping criterion." + Args: + trials: The trials to evaluate the stopping criterion on. + log_status: Whether to log the current status of the budget. + + Returns: + A tuple of (stopping message or False, global resource usage). + """ + worker_resource_usage = self._calculate_total_resource_usage( + trials, + subset_worker_id=self.worker_id, + include_in_progress=self.settings.include_in_progress_evaluations_towards_maximum, + ) + + global_resource_usage = self._calculate_total_resource_usage( + trials, + subset_worker_id=None, + include_in_progress=self.settings.include_in_progress_evaluations_towards_maximum, + ) + + if log_status: + # Log current budget status + budget_info_parts = [] + if self.settings.evaluations_to_spend is not None: + eval_percentage = int( + ( + worker_resource_usage.evaluations + / self.settings.evaluations_to_spend + ) + * 100 + ) + budget_info_parts.append( + "Evaluations:" + f" {worker_resource_usage.evaluations}/" + f"{self.settings.evaluations_to_spend}" + f" ({eval_percentage}%)" ) + if self.settings.fidelities_to_spend is not None: + fidelity_percentage = int( + (worker_resource_usage.fidelities / self.settings.fidelities_to_spend) + * 100 + ) + budget_info_parts.append( + "Fidelities:" + f" {worker_resource_usage.fidelities}/" + f"{self.settings.fidelities_to_spend}" + f" ({fidelity_percentage}%)" + ) + if self.settings.cost_to_spend is not None: + cost_percentage = int( + (worker_resource_usage.cost / self.settings.cost_to_spend) * 100 + ) + budget_info_parts.append( + "Cost:" + f" {worker_resource_usage.cost}/" + f"{self.settings.cost_to_spend} ({cost_percentage}%)" + ) + if self.settings.max_evaluation_time_total_seconds is not None: + time_percentage = int( + ( + worker_resource_usage.time + / self.settings.max_evaluation_time_total_seconds + ) + * 100 + ) + budget_info_parts.append( + "Time:" + f" {worker_resource_usage.time}/" + f"{self.settings.max_evaluation_time_total_seconds}s" + f" ({time_percentage}%)" + ) + + if budget_info_parts: + logger.info("Budget status - %s", " | ".join(budget_info_parts)) + return_string: str | Literal[False] = False - if self.settings.fidelities_to_spend is not None and hasattr( - self.optimizer, "space" + if ( + self.settings.evaluations_to_spend is not None + and worker_resource_usage.evaluations >= self.settings.evaluations_to_spend ): - fidelity_name = next(iter(self.optimizer.space.fidelities.keys())) - count = sum( - trial.config[fidelity_name] - for _, trial in worker_trials.items() - if trial.report is not None and trial.config[fidelity_name] is not None + return_string = ( + "Worker has reached the maximum number of evaluations it is allowed" + f" to do as given by `{self.settings.evaluations_to_spend=}`." + "\nTo allow more evaluations, increase this value or use a different" + " stopping criterion." ) - if count >= self.settings.fidelities_to_spend: - return ( - "The total number of fidelity evaluations has reached the maximum" - f" allowed of `{self.settings.fidelities_to_spend=}`." - " To allow more evaluations, increase this value or use a different" - " stopping criterion." - ) - if self.settings.cost_to_spend is not None: - cost = sum( - trial.report.cost - for _, trial in worker_trials.items() - if trial.report is not None and trial.report.cost is not None + if ( + self.settings.fidelities_to_spend is not None + and worker_resource_usage.fidelities >= self.settings.fidelities_to_spend + ): + return_string = ( + "The total number of fidelity evaluations has reached the maximum" + f" allowed of `{self.settings.fidelities_to_spend=}`." + " To allow more evaluations, increase this value or use a different" + " stopping criterion." ) - if cost >= self.settings.cost_to_spend: - return ( - "Worker has reached the maximum cost it is allowed to spend" - f" which is given by `{self.settings.cost_to_spend=}`." - f" This worker has spend '{cost}'." - "\n To allow more evaluations, increase this value or use a different" - " stopping criterion." - ) - if self.settings.max_evaluation_time_total_seconds is not None: - time_spent = sum( - trial.report.evaluation_duration - for _, trial in worker_trials.items() - if trial.report is not None - if trial.report.evaluation_duration is not None + if ( + self.settings.cost_to_spend is not None + and worker_resource_usage.cost >= self.settings.cost_to_spend + ): + return_string = ( + "Worker has reached the maximum cost it is allowed to spend" + f" which is given by `{self.settings.cost_to_spend=}`." + f" This worker has spend '{worker_resource_usage.cost}'." + "\n To allow more evaluations, increase this value or use a different" + " stopping criterion." ) - if time_spent >= self.settings.max_evaluation_time_total_seconds: - return ( - "The maximum evaluation time of" - f" `{self.settings.max_evaluation_time_total_seconds=}` has been" - " reached. To allow more evaluations, increase this value or use" - " a different stopping criterion." - ) - return False + if ( + self.settings.max_evaluation_time_total_seconds is not None + and worker_resource_usage.time + >= self.settings.max_evaluation_time_total_seconds + ): + return_string = ( + "The maximum evaluation time of" + f" `{self.settings.max_evaluation_time_total_seconds=}` has been" + " reached. To allow more evaluations, increase this value or use" + " a different stopping criterion." + ) + + return (return_string, global_resource_usage) @property def _requires_global_stopping_criterion(self) -> bool: @@ -371,6 +521,34 @@ def _requires_global_stopping_criterion(self) -> bool: or self.settings.max_evaluation_time_total_seconds is not None ) + def _write_trajectory_files( + self, + incumbent_configs: list, + optimal_configs: list, + trace_lock: FileLock, + improvement_trace_path: Path, + best_config_path: Path, + final_stopping_criteria: ResourceUsage | None = None, + ) -> None: + """Writes the trajectory and best config files safely.""" + trace_text = _build_incumbent_content(incumbent_configs) + + best_config_text = _build_optimal_set_content(optimal_configs) + + if final_stopping_criteria: + best_config_text += "\n" + "-" * 80 + best_config_text += "\nFinal cumulative metrics (Assuming completed run):" + for metric, value in final_stopping_criteria.to_trajectory_dict().items(): + best_config_text += f"\n{metric}: {value}" + + with trace_lock: + if incumbent_configs: + with improvement_trace_path.open(mode="w") as f: + f.write(trace_text) + if optimal_configs: + with best_config_path.open(mode="w") as f: + f.write(best_config_text) + def _get_next_trial(self) -> Trial | Literal["break"]: # If there are no global stopping criterion, we can no just return early. with self.state._optimizer_lock.lock(worker_id=self.worker_id): @@ -388,7 +566,10 @@ def _get_next_trial(self) -> Trial | Literal["break"]: trials = self.state._trial_repo.latest() if self._requires_global_stopping_criterion: - should_stop = self._check_global_stopping_criterion(trials) + should_stop, stop_criteria = self._check_global_stopping_criterion( + trials, + log_status=True, + ) if should_stop is not False: logger.info(should_stop) return "break" @@ -488,17 +669,8 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 _set_workers_neps_state(self.state) main_dir = Path(self.state.path) - full_df_path, short_path, csv_locker = _initiate_summary_csv(main_dir) - - # Create empty CSV files - with csv_locker.lock(): - full_df_path.parent.mkdir(parents=True, exist_ok=True) - full_df_path.touch(exist_ok=True) - short_path.touch(exist_ok=True) - summary_dir = main_dir / "summary" summary_dir.mkdir(parents=True, exist_ok=True) - improvement_trace_path = summary_dir / "best_config_trajectory.txt" improvement_trace_path.touch(exist_ok=True) best_config_path = summary_dir / "best_config.txt" @@ -506,31 +678,20 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 _trace_lock = FileLock(".trace.lock") _trace_lock_path = Path(str(_trace_lock.lock_file)) _trace_lock_path.touch(exist_ok=True) + full_df_path, short_path, csv_locker = _initiate_summary_csv(main_dir) + + # Create empty CSV files + with csv_locker.lock(): + full_df_path.parent.mkdir(parents=True, exist_ok=True) + full_df_path.touch(exist_ok=True) + short_path.touch(exist_ok=True) logger.info( "Summary files can be found in the “summary” folder inside" - "the root directory: %s", + " the root directory: %s", summary_dir, ) - previous_trials = self.state.lock_and_read_trials() - if len(previous_trials): - load_incumbent_trace( - previous_trials, - _trace_lock, - self.state, - self.settings, - improvement_trace_path, - best_config_path, - ) - - _best_score_so_far = float("inf") - if ( - self.state.new_score is not None - and self.state.new_score != _best_score_so_far - ): - _best_score_so_far = self.state.new_score - optimizer_name = self.state._optimizer_info["name"] logger.info("Using optimizer: %s", optimizer_name) @@ -662,58 +823,16 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 for _key, callback in _TRIAL_END_CALLBACKS.items(): callback(trial_to_eval) - if ( - report.objective_to_minimize is not None - and report.err is None - and not isinstance(report.objective_to_minimize, list) - ): - self.state.new_score = report.objective_to_minimize - if self.state.new_score < _best_score_so_far: - _best_score_so_far = self.state.new_score - logger.info( - "New best: trial %s with objective %s", - evaluated_trial.id, - self.state.new_score, - ) - - # Store in memory for later file re-writing - self.state.all_best_configs.append( - { - "score": self.state.new_score, - "trial_id": evaluated_trial.id, - "config": evaluated_trial.config, - } + if report.objective_to_minimize is not None and report.err is None: + with self.state._trial_lock.lock(): + evaluated_trials = self.state._trial_repo.get_valid_evaluated_trials() + self.load_incumbent_trace( + evaluated_trials, + _trace_lock, + improvement_trace_path, + best_config_path, ) - # Build trace text and best config text - trace_text = ( - "Best configs and their objectives across evaluations:\n" - + "-" * 80 - + "\n" - ) - for best in self.state.all_best_configs: - trace_text += ( - f"Objective to minimize: {best['score']}\n" - f"Config ID: {best['trial_id']}\n" - f"Config: {best['config']}\n" + "-" * 80 + "\n" - ) - - best_config = self.state.all_best_configs[-1] # Latest best - best_config_text = ( - f"# Best config:" - f"\n\n Config ID: {best_config['trial_id']}" - f"\n Objective to minimize: {best_config['score']}" - f"\n Config: {best_config['config']}" - ) - - # Write files from scratch - with _trace_lock: - with improvement_trace_path.open(mode="w") as f: - f.write(trace_text) - - with best_config_path.open(mode="w") as f: - f.write(best_config_text) - full_df, short = status(main_dir) with csv_locker.lock(): full_df.to_csv(full_df_path) @@ -726,60 +845,83 @@ def run(self) -> None: # noqa: C901, PLR0912, PLR0915 "Learning Curve %s: %s", evaluated_trial.id, report.learning_curve ) + def load_incumbent_trace( + self, + trials: dict[str, Trial], + _trace_lock: FileLock, + improvement_trace_path: Path, + best_config_path: Path, + ) -> None: + """Load the incumbent trace from previous trials and update the state. + This function also computes cumulative resource usage and updates the best + configurations. + + Args: + trials (dict): A dictionary of the evaluated trials which have a valid report. + _trace_lock (FileLock): A file lock to ensure thread-safe writing. + improvement_trace_path (Path): Path to the improvement trace file. + best_config_path (Path): Path to the best configuration file. + """ + if not trials: + return -def load_incumbent_trace( # noqa: D103 - previous_trials: dict[str, Trial], - _trace_lock: FileLock, - state: NePSState, - settings: WorkerSettings, # noqa: ARG001 - improvement_trace_path: Path, - best_config_path: Path, -) -> None: - _best_score_so_far = float("inf") + # Clear any existing entries to prevent duplicates and rebuild a + # non-dominated frontier from previous trials in chronological order. + incumbent = [] - for evaluated_trial in previous_trials.values(): - if ( - evaluated_trial.report is not None - and evaluated_trial.report.objective_to_minimize is not None - ): - state.new_score = evaluated_trial.report.objective_to_minimize - if state.new_score is not None and state.new_score < _best_score_so_far: - _best_score_so_far = state.new_score - state.all_best_configs.append( - { - "score": state.new_score, - "trial_id": evaluated_trial.metadata.id, - "config": evaluated_trial.config, - } - ) + running_usage = ResourceUsage() - trace_text = ( - "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" - ) - for best in state.all_best_configs: - trace_text += ( - f"Objective to minimize: {best['score']}\n" - f"Config ID: {best['trial_id']}\n" - f"Config: {best['config']}\n" + "-" * 80 + "\n" + sorted_trials: list[Trial] = sorted( + trials.values(), + key=lambda t: ( + t.metadata.time_sampled if t.metadata.time_sampled else float("inf") + ), ) - - best_config_text = "" - if state.all_best_configs: - best_config = state.all_best_configs[-1] - best_config_text = ( - f"# Best config:" - f"\n\n Config ID: {best_config['trial_id']}" - f"\n Objective to minimize: {best_config['score']}" - f"\n Config: {best_config['config']}" + is_mo = any( + isinstance(trial.report.objective_to_minimize, list) # type: ignore[union-attr] + for trial in sorted_trials ) - else: - best_config = None - with _trace_lock: - with improvement_trace_path.open(mode="w") as f: - f.write(trace_text) - with best_config_path.open(mode="w") as f: - f.write(best_config_text) + frontier: list[Trial] = [] + trajectory_confs: dict[str, dict[str, float | int]] = {} + + for evaluated_trial in sorted_trials: + single_trial_usage = self._calculate_total_resource_usage( + {evaluated_trial.id: evaluated_trial} + ) + running_usage += single_trial_usage + + assert evaluated_trial.report is not None # for mypy + new_trial_obj = evaluated_trial.report.objective_to_minimize + + if not _is_dominated(new_trial_obj, frontier): + frontier = _prune_and_add_to_frontier(evaluated_trial, frontier) + if not is_mo: + incumbent.append(evaluated_trial) + current_snapshot = ResourceUsage(**asdict(running_usage)) + config_dict = { + "score": new_trial_obj, + "trial_id": evaluated_trial.id, + "config": evaluated_trial.config, + } + if evaluated_trial.report.cost is not None: + config_dict["cost"] = evaluated_trial.report.cost + + config_dict.update(current_snapshot.to_trajectory_dict()) + trajectory_confs[evaluated_trial.id] = config_dict + + optimal_configs: list[dict] = [trajectory_confs[trial.id] for trial in frontier] + incumbent_configs: list[dict] = [ + trajectory_confs[trial.id] for trial in incumbent + ] + + self._write_trajectory_files( + incumbent_configs=incumbent_configs, + optimal_configs=optimal_configs, + trace_lock=_trace_lock, + improvement_trace_path=improvement_trace_path, + best_config_path=best_config_path, + ) def _save_results( @@ -813,9 +955,11 @@ def _save_results( raise RuntimeError(f"Trial '{trial_id}' not found in '{root_directory}'") report = trial.set_complete( - report_as=Trial.State.SUCCESS.value - if result.exception is None - else Trial.State.CRASHED.value, + report_as=( + Trial.State.SUCCESS.value + if result.exception is None + else Trial.State.CRASHED.value + ), objective_to_minimize=result.objective_to_minimize, cost=result.cost, learning_curve=result.learning_curve, @@ -909,6 +1053,7 @@ def _launch_runtime( # noqa: PLR0913 optimizer: AskFunction, optimizer_info: OptimizerInfo, optimization_dir: Path, + pipeline_space: SearchSpace | PipelineSpace, cost_to_spend: float | None, ignore_errors: bool = False, objective_value_on_error: float | None, @@ -961,8 +1106,13 @@ def _launch_runtime( # noqa: PLR0913 shared_state=None, # TODO: Unused for the time being... worker_ids=None, ), + pipeline_space=pipeline_space, ) break + except NePSError: + # Don't retry on NePSError - these are user errors + # like pipeline space mismatch + raise except Exception: # noqa: BLE001 time.sleep(0.5) logger.debug( @@ -1037,3 +1187,59 @@ def _make_default_report_values( learning_curve_on_error=None, learning_curve_if_not_provided="objective_to_minimize", ) + + +def _to_sequence(score: float | Sequence[float]) -> list[float]: + """Normalize score to a list of floats for pareto comparisons. + + Scalars become single-element lists. Sequences are converted to lists. + """ + if isinstance(score, Sequence): + return [float(x) for x in score] + return [float(score)] + + +def _is_dominated(candidate: float | Sequence[float], frontier: list[Trial]) -> bool: + """Return True if `candidate` is dominated by any point in `frontier`. + + `frontier` is a list of score sequences (as lists). + """ + cand_seq = _to_sequence(candidate) + + for t in frontier: + if t.report is None: + continue + f_seq = _to_sequence(t.report.objective_to_minimize) + if len(f_seq) != len(cand_seq): + continue + if all(fi <= ci for fi, ci in zip(f_seq, cand_seq, strict=False)) and any( + fi < ci for fi, ci in zip(f_seq, cand_seq, strict=False) + ): + return True + return False + + +def _prune_and_add_to_frontier(candidate: Trial, frontier: list[Trial]) -> list[Trial]: + """Add candidate Trial to frontier and remove frontier Trials dominated by it. + + Frontier is a list of Trial objects (with reports). Returns the new frontier + as a list of Trials. + """ + if candidate.report is None: + return frontier + + cand_seq = _to_sequence(candidate.report.objective_to_minimize) + new_frontier: list[Trial] = [] + for t in frontier: + if t.report is None: + continue + f_seq = _to_sequence(t.report.objective_to_minimize) + if ( + len(f_seq) == len(cand_seq) + and all(ci <= fi for ci, fi in zip(cand_seq, f_seq, strict=False)) + and any(ci < fi for ci, fi in zip(cand_seq, f_seq, strict=False)) + ): + continue + new_frontier.append(t) + new_frontier.append(candidate) + return new_frontier diff --git a/neps/sampling/priors.py b/neps/sampling/priors.py index dfff8d612..a8ee364ed 100644 --- a/neps/sampling/priors.py +++ b/neps/sampling/priors.py @@ -23,11 +23,13 @@ TruncatedNormal, ) from neps.sampling.samplers import Sampler -from neps.space import Categorical, ConfigEncoder, Domain, Float, Integer +from neps.space import ConfigEncoder, Domain, HPOCategorical, HPOFloat, HPOInteger if TYPE_CHECKING: from torch.distributions import Distribution +PRIOR_CONFIDENCE_MAPPING = {"low": 0.25, "medium": 0.5, "high": 0.75} + class Prior(Sampler): """A protocol for priors over search spaces. @@ -120,7 +122,7 @@ def uniform(cls, ncols: int) -> Uniform: @classmethod def from_parameters( cls, - parameters: Mapping[str, Categorical | Float | Integer], + parameters: Mapping[str, HPOCategorical | HPOFloat | HPOInteger], *, center_values: Mapping[str, Any] | None = None, confidence_values: Mapping[str, float] | None = None, @@ -144,7 +146,7 @@ def from_parameters( Returns: The prior distribution """ - _mapping = {"low": 0.25, "medium": 0.5, "high": 0.75} + _mapping = PRIOR_CONFIDENCE_MAPPING center_values = center_values or {} confidence_values = confidence_values or {} @@ -160,7 +162,9 @@ def from_parameters( continue confidence_score = confidence_values.get(name, _mapping[hp.prior_confidence]) - center = hp.choices.index(default) if isinstance(hp, Categorical) else default + center = ( + hp.choices.index(default) if isinstance(hp, HPOCategorical) else default + ) centers.append((center, confidence_score)) return Prior.from_domains_and_centers(domains=domains, centers=centers) @@ -356,7 +360,7 @@ def log_pdf( if x.shape[-1] != len(self.distributions): raise ValueError( - f"Got a tensor `x` whose last dimesion (the hyperparameter dimension)" + "Got a tensor `x` whose last dimesion (the hyperparameter dimension)" f" is of length {x.shape[-1]=} but" f" the CenteredPrior called has {len(self.distributions)=}" " distributions to use for calculating the `log_pdf`. Perhaps" diff --git a/neps/space/__init__.py b/neps/space/__init__.py index f2bbc55ca..477596241 100644 --- a/neps/space/__init__.py +++ b/neps/space/__init__.py @@ -1,15 +1,21 @@ from neps.space.domain import Domain from neps.space.encoding import ConfigEncoder -from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter +from neps.space.parameters import ( + HPOCategorical, + HPOConstant, + HPOFloat, + HPOInteger, + Parameter, +) from neps.space.search_space import SearchSpace __all__ = [ - "Categorical", "ConfigEncoder", - "Constant", "Domain", - "Float", - "Integer", + "HPOCategorical", + "HPOConstant", + "HPOFloat", + "HPOInteger", "Parameter", "SearchSpace", ] diff --git a/neps/space/encoding.py b/neps/space/encoding.py index d58c63dc3..b0a8527f9 100644 --- a/neps/space/encoding.py +++ b/neps/space/encoding.py @@ -15,7 +15,7 @@ import torch from neps.space.domain import Domain -from neps.space.parameters import Categorical, Float, Integer, Parameter +from neps.space.parameters import HPOCategorical, HPOFloat, HPOInteger, Parameter V = TypeVar("V", int, float) @@ -486,9 +486,9 @@ def from_parameters( continue match hp: - case Float() | Integer(): + case HPOFloat() | HPOInteger(): transformers[name] = MinMaxNormalizer(hp.domain) # type: ignore - case Categorical(): + case HPOCategorical(): transformers[name] = CategoricalToIntegerTransformer(hp.choices) case _: raise ValueError(f"Unsupported parameter type: {type(hp)}.") diff --git a/neps/space/neps_spaces/__init__.py b/neps/space/neps_spaces/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/neps/space/neps_spaces/neps_space.py b/neps/space/neps_spaces/neps_space.py new file mode 100644 index 000000000..a7e641904 --- /dev/null +++ b/neps/space/neps_spaces/neps_space.py @@ -0,0 +1,1409 @@ +"""This module provides functionality for resolving NePS spaces, including sampling from +domains, resolving pipelines, and handling various resolvable objects. +""" + +from __future__ import annotations + +import contextlib +import dataclasses +import functools +from collections.abc import Callable, Generator, Mapping +from functools import partial +from typing import TYPE_CHECKING, Any, Concatenate, Literal, TypeVar, cast + +import neps +from neps.optimizers import algorithms, optimizer +from neps.space.neps_spaces.parameters import ( + _UNSET, + ByName, + Categorical, + Domain, + Fidelity, + Float, + Integer, + Lazy, + Operation, + PipelineSpace, + Repeated, + Resample, + Resolvable, +) +from neps.space.neps_spaces.sampling import ( + DomainSampler, + IOSampler, + OnlyPredefinedValuesSampler, + RandomSampler, +) +from neps.space.parsing import convert_mapping + +if TYPE_CHECKING: + from neps.space import SearchSpace + from neps.state.pipeline_eval import EvaluatePipelineReturn + +P = TypeVar("P", bound="PipelineSpace") + + +def construct_sampling_path( + path_parts: list[str], + domain_obj: Domain, +) -> str: + """Construct a sampling path for a domain object. + + The sampling path uniquely identifies a sampled value in the resolution context. + It consists of the hierarchical path through the pipeline space and a domain + identifier that includes type and range information. + + Args: + path_parts: The hierarchical path parts (e.g., ["Resolvable", "integer1"]). + domain_obj: The domain object for which to construct the path. + + Returns: + A string representing the full sampling path in the format: + "::__" + Example: "Resolvable.integer1::integer__0_1_False" + + Raises: + ValueError: If path_parts is empty or domain_obj is not a Domain. + """ + if not path_parts: + raise ValueError("path_parts cannot be empty") + if not isinstance(domain_obj, Domain): + raise ValueError(f"domain_obj must be a Domain, got {type(domain_obj)}") + + # Get the domain type name (e.g., "integer", "float", "categorical") + domain_obj_type_name = type(domain_obj).__name__.lower() + + # Get the range compatibility identifier (e.g., "0_1_False" for + # Integer(0, 1, log=False)) + range_compatibility_identifier = domain_obj.range_compatibility_identifier + + # Combine type and range: "integer__0_1_False" + domain_obj_identifier = f"{domain_obj_type_name}__{range_compatibility_identifier}" + + # Join path parts with dots: "Resolvable.integer1" + current_path = ".".join(path_parts) + + # Append domain identifier: "Resolvable.integer1::integer__0_1_False" + current_path += "::" + domain_obj_identifier + + return current_path + + +class SamplingResolutionContext: + """A context for resolving samplings in a NePS space. + It manages the resolution root, domain sampler, environment values, + and keeps track of samplings made and resolved objects. + + Args: + resolution_root: The root of the resolution, which should be a Resolvable + object. + domain_sampler: The DomainSampler to use for sampling from Domain objects. + environment_values: A mapping of environment values that are fixed and not + related to samplings. These values can be used in the resolution process. + + Raises: + ValueError: If the resolution_root is not a Resolvable, or if the domain_sampler + is not a DomainSampler, or if the environment_values is not a Mapping. + """ + + def __init__( + self, + *, + resolution_root: Resolvable, + domain_sampler: DomainSampler, + environment_values: Mapping[str, Any], + ): + """Initialize the SamplingResolutionContext with a resolution root, domain + sampler, and environment values. + + Args: + resolution_root: The root of the resolution, which should be a Resolvable + object. + domain_sampler: The DomainSampler to use for sampling from Domain objects. + environment_values: A mapping of environment values that are fixed and not + related to samplings. These values can be used in the resolution process. + + Raises: + ValueError: If the resolution_root is not a Resolvable, or if the + domain_sampler is not a DomainSampler, or if the environment_values is + not a Mapping. + """ + if not isinstance(resolution_root, Resolvable): + raise ValueError( + "The received `resolution_root` is not a Resolvable:" + f" {resolution_root!r}." + ) + + if not isinstance(domain_sampler, DomainSampler): + raise ValueError( + "The received `domain_sampler` is not a DomainSampler:" + f" {domain_sampler!r}." + ) + + if not isinstance(environment_values, Mapping): + raise ValueError( + "The received `environment_values` is not a Mapping:" + f" {environment_values!r}." + ) + + # `_resolution_root` stores the root of the resolution. + self._resolution_root: Resolvable = resolution_root + + # `_domain_sampler` stores the object responsible for sampling from Domain + # objects. + self._domain_sampler = domain_sampler + + # # `_environment_values` stores fixed values from outside. + # # They are not related to samplings and can not be mutated or similar. + self._environment_values = environment_values + + # `_samplings_made` stores the values we have sampled + # and can be used later in case we want to redo a resolving. + self._samplings_made: dict[str, Any] = {} + + # `_resolved_objects` stores the intermediate values to make re-use possible. + self._resolved_objects: dict[Any, Any] = {} + + # `_current_path_parts` stores the current path we are resolving. + self._current_path_parts: list[str] = [] + + @property + def resolution_root(self) -> Resolvable: + """Get the root of the resolution. + + Returns: + The root of the resolution, which should be a Resolvable object. + """ + return self._resolution_root + + @property + def samplings_made(self) -> Mapping[str, Any]: + """Get the samplings made during the resolution process. + + Returns: + A mapping of paths to sampled values. + """ + return self._samplings_made + + @property + def environment_values(self) -> Mapping[str, Any]: + """Get the environment values that are fixed and not related to samplings. + + Returns: + A mapping of environment variable names to their values. + """ + return self._environment_values + + @contextlib.contextmanager + def resolving(self, _obj: Any, name: str) -> Generator[None]: + """Context manager for resolving an object in the current resolution context. + + Args: + _obj: The object being resolved, can be any type. + name: The name of the object being resolved, used for debugging. + + Raises: + ValueError: If the name is not a valid string. + """ + if not name or not isinstance(name, str): + raise ValueError( + f"Given name for what we are resolving is invalid: {name!r}." + ) + + # It is possible that the received object has already been resolved. + # That is expected and is okay, so no check is made for it. + # For example, in the case of a Resample we can receive the same object again. + + self._current_path_parts.append(name) + try: + yield + finally: + self._current_path_parts.pop() + + def was_already_resolved(self, obj: Any) -> bool: + """Check if the given object was already resolved in the current context. + + Args: + obj: The object to check if it was already resolved. + + Returns: + True if the object was already resolved, False otherwise. + """ + return obj in self._resolved_objects + + def add_resolved(self, original: Any, resolved: Any) -> None: + """Add a resolved object to the context. + + Args: + original: The original object that was resolved. + resolved: The resolved value of the original object. + + Raises: + ValueError: If the original object was already resolved or if it is a + Resample. + """ + if self.was_already_resolved(original): + raise ValueError( + f"Original object has already been resolved: {original!r}. " + + "\nIf you are doing resampling by name, " + + "make sure you are not forgetting to request resampling also for" + " related objects." + "\nOtherwise it could lead to infinite recursion." + ) + if isinstance(original, Resample): + raise ValueError( + f"Attempting to add a Resample object to resolved values: {original!r}." + ) + self._resolved_objects[original] = resolved + + def get_resolved(self, obj: Any) -> Any: + """Get the resolved value for the given object. + + Args: + obj: The object for which to get the resolved value. + + Returns: + The resolved value of the object. + + Raises: + ValueError: If the object was not already resolved in the context. + """ + try: + return self._resolved_objects[obj] + except KeyError as err: + raise ValueError( + f"Given object was not already resolved. Please check first: {obj!r}" + ) from err + + def sample_from(self, domain_obj: Domain) -> Any: + """Sample a value from the given domain object. + + Args: + domain_obj: The domain object from which to sample a value. + + Returns: + The sampled value from the domain object. + + Raises: + ValueError: If the domain object was already resolved or if the path + has already been sampled from. + """ + # Each `domain_obj` is only ever sampled from once. + # This is okay and the expected behavior. + # For each `domain_obj`, its sampled value is either directly stored itself, + # or is used in some other Resolvable. + # In both cases that sampled value is cached for later uses, + # and so the `domain_obj` will not be re-sampled from again. + if self.was_already_resolved(domain_obj): + raise ValueError( + "We have already sampled a value for the given domain object:" + f" {domain_obj!r}." + "\nThis should not be happening." + ) + + # Construct the unique sampling path for this domain object + current_path = construct_sampling_path( + path_parts=self._current_path_parts, + domain_obj=domain_obj, + ) + + if current_path in self._samplings_made: + # We have already sampled a value for this path. This should not happen. + # Every time we sample a domain, it should have its own different path. + raise ValueError( + f"We have already sampled a value for the current path: {current_path!r}." + + "\nThis should not be happening." + ) + + sampled_value = self._domain_sampler( + domain_obj=domain_obj, + current_path=current_path, + ) + + self._samplings_made[current_path] = sampled_value + return self._samplings_made[current_path] + + def get_value_from_environment(self, var_name: str) -> Any: + """Get a value from the environment variables. + + Args: + var_name: The name of the environment variable to get the value from. + + Returns: + The value of the environment variable. + + Raises: + ValueError: If the environment variable is not found in the context. + """ + try: + return self._environment_values[var_name] + except KeyError as err: + raise ValueError( + f"No value is available for the environment variable {var_name!r}." + ) from err + + +class SamplingResolver: + """A class responsible for resolving samplings in a NePS space. + It uses a SamplingResolutionContext to manage the resolution process, + and a DomainSampler to sample values from Domain objects. + """ + + def __call__( + self, + obj: Resolvable, + domain_sampler: DomainSampler, + environment_values: Mapping[str, Any], + ) -> tuple[Resolvable, SamplingResolutionContext]: + """Resolve the given object in the context of the provided domain sampler and + environment values. + + Args: + obj: The Resolvable object to resolve. + domain_sampler: The DomainSampler to use for sampling from Domain objects. + environment_values: A mapping of environment values that are fixed and not + related to samplings. + + Returns: + A tuple containing the resolved object and the + SamplingResolutionContext. + + Raises: + ValueError: If the object is not a Resolvable, or if the domain_sampler + is not a DomainSampler, or if the environment_values is not a Mapping. + """ + context = SamplingResolutionContext( + resolution_root=obj, + domain_sampler=domain_sampler, + environment_values=environment_values, + ) + return self._resolve(obj, "Resolvable", context), context + + def _resolve(self, obj: Any, name: str, context: SamplingResolutionContext) -> Any: + with context.resolving(obj, name): + return self._resolver_dispatch(obj, context) + + @functools.singledispatchmethod + def _resolver_dispatch( + self, + any_obj: Any, + _context: SamplingResolutionContext, + ) -> Any: + # Default resolver. To be used for types which are not instances of `Resolvable`. + # No need to store or lookup from context, directly return the given object. + if isinstance(any_obj, Resolvable): + raise ValueError( + "The default resolver is not supposed to be called for resolvable" + f" objects. Received: {any_obj!r}." + ) + return any_obj + + @_resolver_dispatch.register + def _( + self, + pipeline_obj: PipelineSpace, + context: SamplingResolutionContext, + ) -> Any: + if context.was_already_resolved(pipeline_obj): + return context.get_resolved(pipeline_obj) + + initial_attrs = pipeline_obj.get_attrs() + final_attrs = {} + needed_resolving = False + + for attr_name, initial_attr_value in initial_attrs.items(): + resolved_attr_value = self._resolve(initial_attr_value, attr_name, context) + final_attrs[attr_name] = resolved_attr_value + needed_resolving = needed_resolving or ( + initial_attr_value is not resolved_attr_value + ) + + result = pipeline_obj + if needed_resolving: + result = pipeline_obj.from_attrs(final_attrs) + + context.add_resolved(pipeline_obj, result) + return result + + @_resolver_dispatch.register + def _( + self, + domain_obj: Domain, + context: SamplingResolutionContext, + ) -> Any: + if context.was_already_resolved(domain_obj): + return context.get_resolved(domain_obj) + + initial_attrs = domain_obj.get_attrs() + final_attrs = {} + needed_resolving = False + + for attr_name, initial_attr_value in initial_attrs.items(): + resolved_attr_value = self._resolve(initial_attr_value, attr_name, context) + final_attrs[attr_name] = resolved_attr_value + needed_resolving = needed_resolving or ( + initial_attr_value is not resolved_attr_value + ) + + resolved_domain_obj = domain_obj + if needed_resolving: + resolved_domain_obj = domain_obj.from_attrs(final_attrs) + + try: + sampled_value = context.sample_from(resolved_domain_obj) + except Exception as e: + raise ValueError(f"Failed to sample from {resolved_domain_obj!r}.") from e + result = self._resolve(sampled_value, "sampled_value", context) + + context.add_resolved(domain_obj, result) + return result + + @_resolver_dispatch.register + def _( + self, + categorical_obj: Categorical, + context: SamplingResolutionContext, + ) -> Any: + if context.was_already_resolved(categorical_obj): + return context.get_resolved(categorical_obj) + + # In the case of categorical choices, we may skip resolving each choice initially, + # only after sampling we go into resolving whatever choice was chosen. + # This avoids resolving things which won't be needed at all. + # If the choices themselves come from some Resolvable, they will be resolved. + + initial_attrs = categorical_obj.get_attrs() + final_attrs = {} + needed_resolving = False + + for attr_name, initial_attr_value in initial_attrs.items(): + if attr_name == "choices": + # We need special handling if we are dealing with a "choice provider", + # which will select a tuple of choices from its own choices, + # from which then this original categorical will pick. + + # Ideally, from the choices provided, we want to first pick one, + # and then only resolve that picked item. + # We don't want the resolution process to directly go inside + # the tuple of provided choices that gets picked from the provider, + # since that would lead to potentially exponential growth + # and in resolving stuff that will ultimately be useless to us. + + # For this reason, if we haven't already sampled this categorical + # (the choice provider), we make sure to wrap each of the choices + # inside it in a lazy resolvable. + # This ensures that the resolving process stops directly after + # the provider has made its choice. + + # Since we may be manually creating a new categorical object + # for the provider, which is what will then get resolved, + # it's important that we manually store + # in the context that resolved value for the original object. + # The original object can possibly be reused elsewhere. + + if isinstance( + initial_attr_value, Categorical + ) and context.was_already_resolved(initial_attr_value): + # Before making adjustments, we make sure we haven't + # already chosen a value for the provider. + # Otherwise, we already have the final answer for it. + resolved_attr_value = context.get_resolved(initial_attr_value) + elif isinstance(initial_attr_value, Categorical) or ( + isinstance(initial_attr_value, Resample) + and isinstance(initial_attr_value.source, Categorical) + ): + # We have a previously unseen provider. + # Create a new object where the choices are lazy, + # and then sample from it, manually tracking the context. + + choice_provider_final_attrs = {**initial_attr_value.get_attrs()} + choice_provider_choices = choice_provider_final_attrs["choices"] + if isinstance(choice_provider_choices, tuple | list): + choice_provider_choices = tuple( + Lazy(content=choice) for choice in choice_provider_choices + ) + choice_provider_final_attrs["choices"] = choice_provider_choices + choice_provider_adjusted = initial_attr_value.from_attrs( + choice_provider_final_attrs + ) + + resolved_attr_value = self._resolve( + choice_provider_adjusted, "choice_provider", context + ) + if not isinstance(initial_attr_value, Resample): + # It's important that we handle filling the context here, + # as we manually created a different object from the original. + # In case the original categorical is used again, + # it will need to be reused with the final value we resolved. + context.add_resolved(initial_attr_value, resolved_attr_value) + else: + # We have "choices" which are ready to use. + resolved_attr_value = initial_attr_value + else: + resolved_attr_value = self._resolve( + initial_attr_value, attr_name, context + ) + final_attrs[attr_name] = resolved_attr_value + needed_resolving = needed_resolving or ( + initial_attr_value is not resolved_attr_value + ) + + resolved_categorical_obj = categorical_obj + if needed_resolving: + resolved_categorical_obj = cast( + Categorical, categorical_obj.from_attrs(final_attrs) + ) + + try: + sampled_index = context.sample_from(resolved_categorical_obj) + except Exception as e: + raise ValueError( + f"Failed to sample from {resolved_categorical_obj!r}." + ) from e + sampled_value = cast(tuple, resolved_categorical_obj.choices)[sampled_index] + result = self._resolve(sampled_value, "sampled_value", context) + + context.add_resolved(categorical_obj, result) + return result + + @_resolver_dispatch.register + def _( + self, + operation_obj: Operation, + context: SamplingResolutionContext, + ) -> Any: + if context.was_already_resolved(operation_obj): + return context.get_resolved(operation_obj) + + initial_attrs = operation_obj.get_attrs() + final_attrs = {} + needed_resolving = False + + for attr_name, initial_attr_value in initial_attrs.items(): + resolved_attr_value = self._resolve(initial_attr_value, attr_name, context) + + # Special handling for 'args': if it was a Resolvable that resolved to a + # non-iterable, wrap it in a tuple since Operation expects args to be a + # sequence + if ( + attr_name == "args" + and isinstance(initial_attr_value, Resolvable) + and not isinstance(resolved_attr_value, tuple | list | Resolvable) + ): + resolved_attr_value = (resolved_attr_value,) + + final_attrs[attr_name] = resolved_attr_value + needed_resolving = needed_resolving or ( + initial_attr_value is not resolved_attr_value + ) + + result = operation_obj + if needed_resolving: + result = operation_obj.from_attrs(final_attrs) + + context.add_resolved(operation_obj, result) + return result + + @_resolver_dispatch.register + def _( + self, + resampled_obj: Resample, + context: SamplingResolutionContext, + ) -> Any: + # The results of Resample are never stored or looked up from cache + # since it would break the logic of their expected behavior. + # Particularly, when Resample objects are nested (at any depth) inside of + # other Resample objects, adding them to the resolution context would result + # in the resolution not doing the right thing. + + if resampled_obj.is_resampling_by_name: + # We are dealing with a resampling by name (ByName reference), + # We will first need to look up the source object referenced by name. + # That will then be the object to resample. + by_name_ref = cast(ByName, resampled_obj.source) + referenced_obj_name = by_name_ref.name + referenced_obj = getattr(context.resolution_root, referenced_obj_name) + resampled_obj = referenced_obj.resample() + + initial_attrs = resampled_obj.get_attrs() + resolvable_to_resample_obj = resampled_obj.from_attrs(initial_attrs) + + if resolvable_to_resample_obj is resampled_obj.source: + # The final resolvable we are resolving needs to be a different + # instance from the original wrapped object. + # Otherwise, it's possible we'll be taking its result + # from the context cache, instead of resampling it. + raise ValueError( + "The final object must be a different instance from the original: " + f"{resolvable_to_resample_obj!r}" + ) + + type_name = type(resolvable_to_resample_obj).__name__.lower() + return self._resolve( + resolvable_to_resample_obj, f"resampled_{type_name}", context + ) + + @_resolver_dispatch.register + def _( + self, + fidelity_obj: Fidelity, + context: SamplingResolutionContext, + ) -> Any: + # A Fidelity object should only really be used in one place, + # so we check if we have seen it before. + # For that we will be storing its result in the resolved cache. + if context.was_already_resolved(fidelity_obj): + raise ValueError("Fidelity object reused multiple times in the pipeline.") + + # The way resolution works for Fidelity objects is that + # we use the domain inside it only to know the bounds for valid values. + # The actual value for the fidelity comes from the outside in the form of an + # environment value, which we look up by the attribute name of the + # received fidelity object inside the resolution root. + + names_for_this_fidelity_obj = [ + attr_name + for attr_name, attr_value in context.resolution_root.get_attrs().items() + if attr_value is fidelity_obj + ] + + if len(names_for_this_fidelity_obj) == 0: + raise ValueError( + "A fidelity object should be a direct attribute of the pipeline." + ) + if len(names_for_this_fidelity_obj) > 1: + raise ValueError( + "A fidelity object should only be referenced once in the pipeline." + ) + + fidelity_name = names_for_this_fidelity_obj[0] + + try: + result = context.get_value_from_environment(fidelity_name) + except ValueError as err: + raise ValueError( + "No value is available in the environment for fidelity" + f" {fidelity_name!r}." + ) from err + + if not fidelity_obj.lower <= result <= fidelity_obj.upper: + raise ValueError( + f"Value for fidelity with name {fidelity_name!r} is outside its allowed" + " range " + + f"[{fidelity_obj.lower!r}, {fidelity_obj.upper!r}]. " + + f"Received: {result!r}." + ) + + context.add_resolved(fidelity_obj, result) + return result + + @_resolver_dispatch.register + def _( + self, + repeated_resolvable_obj: Repeated, + context: SamplingResolutionContext, + ) -> tuple[Any]: + if context.was_already_resolved(repeated_resolvable_obj): + return context.get_resolved(repeated_resolvable_obj) + + # First figure out how many times we need to resolvable repeated, + # then do that many resolves of that object. + # It does not matter what type the content is. + # Return all the results as a tuple. + + unresolved_count = repeated_resolvable_obj.count + resolved_count = self._resolve(unresolved_count, "repeat_count", context) + + if not isinstance(resolved_count, int): + raise ValueError( + f"The resolved count value for {repeated_resolvable_obj!r} is not an int." + f" Resolved to {resolved_count!r}" + ) + + obj_to_repeat = repeated_resolvable_obj.content + result = [] + for i in range(resolved_count): + result.append(self._resolve(obj_to_repeat, f"repeated_item[{i}]", context)) + result = tuple(result) # type: ignore[assignment] + + context.add_resolved(repeated_resolvable_obj, result) + return result # type: ignore[return-value] + + @_resolver_dispatch.register + def _( + self, + resolvable_obj: Lazy, + context: SamplingResolutionContext, # noqa: ARG002 + ) -> Any: + # When resolving a lazy resolvable, + # just directly return the content it's holding. + # The purpose of the lazy resolvable is to stop + # the resolver from going deeper into the process. + # In this case, to stop the resolution of `resolvable_obj.content`. + # No need to add it in the resolved cache. + return resolvable_obj.content + + @_resolver_dispatch.register + def _( + self, + by_name_obj: ByName, + context: SamplingResolutionContext, + ) -> Any: + # When resolving a ByName reference directly (not wrapped in Resample), + # look up the referenced parameter and resolve it. + # This is cached so multiple references to the same parameter will + # return the same resolved value. + if context.was_already_resolved(by_name_obj): + return context.get_resolved(by_name_obj) + + referenced_obj_name = by_name_obj.name + referenced_obj = getattr(context.resolution_root, referenced_obj_name) + result = self._resolve(referenced_obj, referenced_obj_name, context) + + context.add_resolved(by_name_obj, result) + return result + + @_resolver_dispatch.register + def _( + self, + resolvable_obj: dict, + context: SamplingResolutionContext, + ) -> dict[Any, Any]: + # The logic below is done so that if the original dict + # had only things that didn't need resolving, + # we return the original object. + # That is important for the rest of the resolving process. + original_dict = resolvable_obj + new_dict = {} + needed_resolving = False + + for k, initial_v in original_dict.items(): + resolved_v = self._resolve(initial_v, f"mapping_value{{{k}}}", context) + new_dict[k] = resolved_v + needed_resolving = needed_resolving or (resolved_v is not initial_v) + + result = original_dict + if needed_resolving: + result = new_dict + + # TODO: [lum] reconsider this below. We likely should cache them, + # similarly to other things. + # IMPORTANT: Dicts are not stored in the resolved cache. + # Otherwise, we won't go inside them the next time + # and will ignore any resampled things inside. + return result + + @_resolver_dispatch.register + def _( + self, + resolvable_obj: tuple, + context: SamplingResolutionContext, + ) -> tuple[Any]: + return self._resolve_sequence(resolvable_obj, context) # type: ignore[return-value] + + @_resolver_dispatch.register + def _( + self, + resolvable_obj: list, + context: SamplingResolutionContext, + ) -> list[Any]: + return self._resolve_sequence(resolvable_obj, context) # type: ignore[return-value] + + def _resolve_sequence( + self, + resolvable_obj: tuple | list, + context: SamplingResolutionContext, + ) -> tuple[Any] | list[Any]: + # The logic below is done so that if the original sequence + # had only things that didn't need resolving, + # we return the original object. + # That is important for the rest of the resolving process. + original_sequence = resolvable_obj + new_list = [] + needed_resolving = False + + for idx, initial_item in enumerate(original_sequence): + resolved_item = self._resolve(initial_item, f"sequence[{idx}]", context) + new_list.append(resolved_item) + needed_resolving = needed_resolving or (initial_item is not resolved_item) + + result = original_sequence + if needed_resolving: + # We also want to return a result of the same type + # as the original received sequence. + original_type = type(original_sequence) + result = original_type(new_list) + + # TODO: [lum] reconsider this below. We likely should cache them, + # similarly to other things. + # IMPORTANT: Sequences are not stored in the resolved cache. + # Otherwise, we won't go inside them the next time + # and will ignore any resampled things inside. + return result + + @_resolver_dispatch.register + def _( + self, + resolvable_obj: Resolvable, + context: SamplingResolutionContext, # noqa: ARG002 + ) -> Any: + # Called when no specialized resolver was available for the specific resolvable + # type. That is not something that is normally expected. + raise ValueError( + "No specialized resolver was registered for object of type" + f" {type(resolvable_obj)!r}." + ) + + +def resolve( + pipeline: P, + domain_sampler: DomainSampler | None = None, + environment_values: Mapping[str, Any] | None = None, +) -> tuple[P, SamplingResolutionContext]: + """Resolve a NePS pipeline with the given domain sampler and environment values. + + Args: + pipeline: The pipeline to resolve, which should be a Pipeline object. + domain_sampler: The DomainSampler to use for sampling from Domain objects. + If None, a RandomSampler with no predefined values will be used. + environment_values: A mapping of environment variable names to their values. + If None, an empty mapping will be used. + + Returns: + A tuple containing the resolved pipeline and the SamplingResolutionContext. + + Raises: + ValueError: If the pipeline is not a Pipeline object or if the domain_sampler + is not a DomainSampler or if the environment_values is not a Mapping. + """ + if domain_sampler is None: + # By default, use a random sampler with no predefined values. + domain_sampler = RandomSampler(predefined_samplings={}) + + if environment_values is None: + # By default, have no environment values. + environment_values = {} + if isinstance(domain_sampler, IOSampler): + environment_values = domain_sampler.sample_environment_values(pipeline) + + sampling_resolver = SamplingResolver() + resolved_pipeline, context = sampling_resolver( + obj=pipeline, + domain_sampler=domain_sampler, + environment_values=environment_values, + ) + return cast(P, resolved_pipeline), context + + +# ------------------------------------------------- + + +def convert_operation_to_callable(operation: Operation) -> Callable: + """Convert an Operation to a callable that can be executed. + + Args: + operation: The Operation to convert. + + Returns: + A callable that represents the operation. + + Raises: + ValueError: If the operation is not a valid Operation object. + """ + operator = cast(Callable, operation.operator) + + operation_args: list[Any] = [] + for arg in operation.args: + if isinstance(arg, tuple | list): + arg_sequence: list[Any] = [] + for a in arg: + converted_arg = ( + convert_operation_to_callable(a) if isinstance(a, Operation) else a + ) + arg_sequence.append(converted_arg) + if isinstance(arg, tuple): + operation_args.append(tuple(arg_sequence)) + else: + operation_args.append(arg_sequence) + else: + operation_args.append( + convert_operation_to_callable(arg) if isinstance(arg, Operation) else arg + ) + + operation_kwargs: dict[str, Any] = {} + for kwarg_name, kwarg_value in operation.kwargs.items(): + if isinstance(kwarg_value, tuple | list): + kwarg_sequence: list[Any] = [] + for a in kwarg_value: + converted_kwarg = ( + convert_operation_to_callable(a) if isinstance(a, Operation) else a + ) + kwarg_sequence.append(converted_kwarg) + if isinstance(kwarg_value, tuple): + operation_kwargs[kwarg_name] = tuple(kwarg_sequence) + else: + operation_kwargs[kwarg_name] = kwarg_sequence + else: + operation_kwargs[kwarg_name] = ( + convert_operation_to_callable(kwarg_value) + if isinstance(kwarg_value, Operation) + else kwarg_value + ) + + return cast(Callable, operator(*operation_args, **operation_kwargs)) + + +# ------------------------------------------------- + + +class NepsCompatConverter: + """A class to convert between NePS configurations and NEPS-compatible configurations. + It provides methods to convert a SamplingResolutionContext to a NEPS-compatible config + and to convert a NEPS-compatible config back to a SamplingResolutionContext. + """ + + _SAMPLING_PREFIX = "SAMPLING__" + _ENVIRONMENT_PREFIX = "ENVIRONMENT__" + _SAMPLING_PREFIX_LEN = len(_SAMPLING_PREFIX) + _ENVIRONMENT_PREFIX_LEN = len(_ENVIRONMENT_PREFIX) + + @dataclasses.dataclass(frozen=True) + class _FromNepsConfigResult: + predefined_samplings: Mapping[str, Any] + environment_values: Mapping[str, Any] + extra_kwargs: Mapping[str, Any] + + @classmethod + def to_neps_config( + cls, + resolution_context: SamplingResolutionContext, + ) -> Mapping[str, Any]: + """Convert a SamplingResolutionContext to a NEPS-compatible config. + + Args: + resolution_context: The SamplingResolutionContext to convert. + + Returns: + A mapping of NEPS-compatible configuration keys to their values. + + Raises: + ValueError: If the resolution_context is not a SamplingResolutionContext. + """ + config: dict[str, Any] = {} + + samplings_made = resolution_context.samplings_made + for sampling_path, value in samplings_made.items(): + config[f"{cls._SAMPLING_PREFIX}{sampling_path}"] = value + + environment_values = resolution_context.environment_values + for env_name, value in environment_values.items(): + config[f"{cls._ENVIRONMENT_PREFIX}{env_name}"] = value + + return config + + @classmethod + def from_neps_config( + cls, + config: Mapping[str, Any], + ) -> _FromNepsConfigResult: + """Convert a NEPS-compatible config to a SamplingResolutionContext. + + Args: + config: A mapping of NEPS-compatible configuration keys to their values. + + Returns: + A _FromNepsConfigResult containing predefined samplings, + environment values, and extra kwargs. + + Raises: + ValueError: If the config is not a valid NEPS-compatible config. + """ + predefined_samplings = {} + environment_values = {} + extra_kwargs = {} + + for name, value in config.items(): + if name.startswith(cls._SAMPLING_PREFIX): + sampling_path = name[cls._SAMPLING_PREFIX_LEN :] + predefined_samplings[sampling_path] = value + elif name.startswith(cls._ENVIRONMENT_PREFIX): + env_name = name[cls._ENVIRONMENT_PREFIX_LEN :] + environment_values[env_name] = value + else: + extra_kwargs[name] = value + + return cls._FromNepsConfigResult( + predefined_samplings=predefined_samplings, + environment_values=environment_values, + extra_kwargs=extra_kwargs, + ) + + +def _prepare_sampled_configs( + chosen_pipelines: list[tuple[PipelineSpace, SamplingResolutionContext]], + n_prev_trials: int, + return_single: bool, # noqa: FBT001 +) -> optimizer.SampledConfig | list[optimizer.SampledConfig]: + configs = [] + for i, (_resolved_pipeline, resolution_context) in enumerate(chosen_pipelines): + neps_config = NepsCompatConverter.to_neps_config( + resolution_context=resolution_context, + ) + + config = optimizer.SampledConfig( + config=neps_config, + id=str(n_prev_trials + i + 1), + previous_config_id=None, + ) + configs.append(config) + + if return_single: + return configs[0] + + return configs + + +def adjust_evaluation_pipeline_for_neps_space( + evaluation_pipeline: Callable[..., EvaluatePipelineReturn], + pipeline_space: P, + operation_converter: Callable[[Operation], Any] = convert_operation_to_callable, +) -> Callable: + """Adjust the evaluation pipeline to work with a NePS space. + This function wraps the evaluation pipeline to sample from the NePS space + and convert the sampled pipeline to a format compatible with the evaluation pipeline. + + Args: + evaluation_pipeline: The evaluation pipeline to adjust. + pipeline_space: The NePS pipeline space to sample from. + operation_converter: A callable to convert Operation objects to a format + compatible with the evaluation pipeline. + + Returns: + A wrapped evaluation pipeline that samples from the NePS space. + + Raises: + ValueError: If the evaluation_pipeline is not callable or if the + pipeline_space is not a Pipeline object. + """ + + @functools.wraps(evaluation_pipeline) + def inner(*args: Any, **kwargs: Any) -> Any: + # `kwargs` can contain other things not related to + # the samplings to make or to environment values. + # That is not an issue. Those items will be passed through. + + sampled_pipeline_data = NepsCompatConverter.from_neps_config(config=kwargs) + + sampled_pipeline, _resolution_context = resolve( + pipeline=pipeline_space, + domain_sampler=OnlyPredefinedValuesSampler( + predefined_samplings=sampled_pipeline_data.predefined_samplings, + ), + environment_values=sampled_pipeline_data.environment_values, + ) + + config = dict(**sampled_pipeline.get_attrs()) + + for name, value in config.items(): + if isinstance(value, Operation): + # If the operator is a not a string, we convert it to a callable. + if isinstance(value.operator, str): + config[name] = value.operator + else: + config[name] = operation_converter(value) + + # So that we still pass the kwargs not related to the config, + # start with the extra kwargs we passed to the converter. + new_kwargs = dict(**sampled_pipeline_data.extra_kwargs) + # Then add all the kwargs from the config. + new_kwargs.update(config) + + return evaluation_pipeline(*args, **new_kwargs) + + return inner + + +def convert_neps_to_classic_search_space(space: PipelineSpace) -> SearchSpace | None: + """Convert a NePS space to a classic SearchSpace if possible. + This function checks if the NePS space can be converted to a classic SearchSpace + by ensuring that it does not contain any complex types like Operation or Resample, + and that all choices of Categorical parameters are of basic types (int, str, float). + If the checks pass, it converts the NePS space to a classic SearchSpace. + + Args: + space: The NePS space to convert, which should be a Pipeline object. + + Returns: + A classic SearchSpace if the conversion is possible, otherwise None. + """ + # First check: No parameters are of type Operation or Resample + if not any( + isinstance(param, Operation | Resample) for param in space.get_attrs().values() + ): + # Second check: All choices of all categoricals are of basic + # types i.e. int, str or float + categoricals = [ + param + for param in space.get_attrs().values() + if isinstance(param, Categorical) + ] + if all( + any( + all(isinstance(choice, datatype) for choice in list(cat_param.choices)) # type: ignore + for datatype in [int, float, str] + ) + for cat_param in categoricals + ): + # If both checks pass, convert the space to a classic SearchSpace + classic_space: dict[str, Any] = {} + for key, value in space.get_attrs().items(): + if isinstance(value, Categorical): + classic_space[key] = neps.HPOCategorical( + choices=list(set(value.choices)), # type: ignore + prior=value.choices[value.prior] if value.has_prior else None, # type: ignore + prior_confidence=( + value.prior_confidence.value if value.has_prior else "low" + ), + ) + elif isinstance(value, Integer): + classic_space[key] = neps.HPOInteger( + lower=value.lower, + upper=value.upper, + log=value._log if hasattr(value, "_log") else False, + prior=value.prior if value.has_prior else None, + prior_confidence=( + value.prior_confidence.value if value.has_prior else "low" + ), + ) + elif isinstance(value, Float): + classic_space[key] = neps.HPOFloat( + lower=value.lower, + upper=value.upper, + log=value._log if hasattr(value, "_log") else False, + prior=value.prior if value.has_prior else None, + prior_confidence=( + value.prior_confidence.value if value.has_prior else "low" + ), + ) + elif isinstance(value, Fidelity): + if isinstance(value.domain, Integer): + classic_space[key] = neps.HPOInteger( + lower=value.domain.lower, + upper=value.domain.upper, + log=( + value.domain._log + if hasattr(value.domain, "_log") + else False + ), + is_fidelity=True, + ) + elif isinstance(value.domain, Float): + classic_space[key] = neps.HPOFloat( + lower=value.domain.lower, + upper=value.domain.upper, + log=( + value.domain._log + if hasattr(value.domain, "_log") + else False + ), + is_fidelity=True, + ) + else: + classic_space[key] = neps.HPOConstant(value) + return convert_mapping(classic_space) + return None + + +def convert_classic_to_neps_search_space( + space: SearchSpace, +) -> PipelineSpace: + """Convert a classic SearchSpace to a NePS PipelineSpace if possible. + This function converts a classic SearchSpace to a NePS PipelineSpace. + + Args: + space: The classic SearchSpace to convert. + + Returns: + A NePS PipelineSpace. + """ + + class NEPSSpace(PipelineSpace): + """A NePS-specific PipelineSpace.""" + + for parameter_name, parameter in space.elements.items(): + if isinstance(parameter, neps.HPOCategorical): + setattr( + NEPSSpace, + parameter_name, + Categorical( + choices=tuple(parameter.choices), + prior=( + parameter.choices.index(parameter.prior) + if parameter.prior + else _UNSET + ), + prior_confidence=( + parameter.prior_confidence + if parameter.prior_confidence + else _UNSET + ), + ), + ) + elif isinstance(parameter, neps.HPOConstant): + setattr(NEPSSpace, parameter_name, parameter.value) + elif isinstance(parameter, neps.HPOInteger): + new_integer = Integer( + lower=parameter.lower, + upper=parameter.upper, + log=parameter.log, + prior=parameter.prior if parameter.prior else _UNSET, + prior_confidence=( + parameter.prior_confidence if parameter.prior_confidence else _UNSET + ), + ) + setattr( + NEPSSpace, + parameter_name, + (Fidelity(domain=new_integer) if parameter.is_fidelity else new_integer), + ) + elif isinstance(parameter, neps.HPOFloat): + new_float = Float( + lower=parameter.lower, + upper=parameter.upper, + log=parameter.log, + prior=parameter.prior if parameter.prior else _UNSET, + prior_confidence=( + parameter.prior_confidence if parameter.prior_confidence else _UNSET + ), + ) + setattr( + NEPSSpace, + parameter_name, + (Fidelity(domain=new_float) if parameter.is_fidelity else new_float), + ) + + return NEPSSpace() + + +ONLY_CLASSIC_ALGORITHMS_NAMES = [ + "asha", + "bayesian_optimization", + "ifbo", + "mo_hyperband", + "primo", + "async_hb", + "successive_halving", + "moasha", + "pibo", +] +CLASSIC_AND_NEPS_ALGORITHMS_NAMES = [ + "random_search", + "priorband", + "hyperband", + "grid_search", +] + + +# Lazy initialization to avoid circular imports +def _get_only_classic_algorithms_functions() -> list[Callable]: + """Get the list of classic-only algorithm functions lazily.""" + return [ + algorithms.asha, + algorithms.bayesian_optimization, + algorithms.ifbo, + algorithms.mo_hyperband, + algorithms.primo, + algorithms.async_hb, + algorithms.successive_halving, + algorithms.moasha, + algorithms.pibo, + ] + + +def _get_classic_and_neps_algorithms_functions() -> list[Callable]: + """Get the list of classic and NEPS algorithm functions lazily.""" + return [ + algorithms.random_search, + algorithms.priorband, + algorithms.hyperband, + algorithms.grid_search, + ] + + +def check_neps_space_compatibility( + optimizer_to_check: ( + algorithms.OptimizerChoice + | Mapping[str, Any] + | tuple[algorithms.OptimizerChoice, Mapping[str, Any]] + | Callable[ + Concatenate[SearchSpace, ...], optimizer.AskFunction + ] # Hack, while we transit + | Callable[ + Concatenate[PipelineSpace, ...], optimizer.AskFunction + ] # from SearchSpace to + | Callable[ + Concatenate[SearchSpace | PipelineSpace, ...], optimizer.AskFunction + ] # Pipeline + | algorithms.CustomOptimizer + | Literal["auto"] + ) = "auto", +) -> Literal["neps", "classic", "both"]: + """Check if the given optimizer is compatible with a NePS space. + This function checks if the optimizer is a NePS-specific algorithm, + a classic algorithm, or a combination of both. + + Args: + optimizer_to_check: The optimizer to check for compatibility. + It can be a NePS-specific algorithm, a classic algorithm, + or a combination of both. + + Returns: + A string indicating the compatibility: + - "neps" if the optimizer is a NePS-specific algorithm, + - "classic" if the optimizer is a classic algorithm, + - "both" if the optimizer is a combination of both. + """ + inner_optimizer = None + if isinstance(optimizer_to_check, partial): + inner_optimizer = optimizer_to_check.func + while isinstance(inner_optimizer, partial): + inner_optimizer = inner_optimizer.func + + only_classic_algorithm = ( + optimizer_to_check in _get_only_classic_algorithms_functions() + or ( + inner_optimizer + and inner_optimizer in _get_only_classic_algorithms_functions() + ) + or ( + optimizer_to_check[0] in ONLY_CLASSIC_ALGORITHMS_NAMES + if isinstance(optimizer_to_check, tuple) + else False + ) + or ( + optimizer_to_check in ONLY_CLASSIC_ALGORITHMS_NAMES + if isinstance(optimizer_to_check, str) + else False + ) + ) + if only_classic_algorithm: + return "classic" + neps_and_classic_algorithm = ( + optimizer_to_check in _get_classic_and_neps_algorithms_functions() + or ( + inner_optimizer + and inner_optimizer in _get_classic_and_neps_algorithms_functions() + ) + or optimizer_to_check == "auto" + or ( + optimizer_to_check[0] in CLASSIC_AND_NEPS_ALGORITHMS_NAMES + if isinstance(optimizer_to_check, tuple) + else False + ) + or ( + optimizer_to_check in CLASSIC_AND_NEPS_ALGORITHMS_NAMES + if isinstance(optimizer_to_check, str) + else False + ) + ) + if neps_and_classic_algorithm: + return "both" + return "neps" diff --git a/neps/space/neps_spaces/parameters.py b/neps/space/neps_spaces/parameters.py new file mode 100644 index 000000000..98c3134f9 --- /dev/null +++ b/neps/space/neps_spaces/parameters.py @@ -0,0 +1,2041 @@ +"""This module defines various classes and protocols for representing and manipulating +search spaces in NePS (Neural Parameter Search). It includes definitions for domains, +pipelines, operations, and fidelity, as well as utilities for sampling and resolving +search spaces. +""" + +from __future__ import annotations + +import abc +import enum +import importlib +import logging +import math +import random +import warnings +from collections.abc import Callable, Mapping, Sequence +from typing import ( + Any, + Generic, + Literal, + Protocol, + TypeVar, + cast, + runtime_checkable, +) + +T = TypeVar("T") + +# Shared docstring constants for DRY +_RESAMPLE_DOCSTRING = """Wrap this {type_name} in a Resample container. + +This allows resampling the {type_name} each time it's resolved, useful for +creating dynamic structures where the same {description} is sampled multiple +times independently. + +Returns: + A Resample instance wrapping this {type_name}. + +Example: + ```python + # Instead of: neps.Resample(my_{type_lower}) + # You can write: my_{type_lower}.resample() + ``` +""" + +_COMPARE_DOMAIN_DOCSTRING = """Check if this {type_name} parameter is equivalent to \ +another. + +This method provides comparison logic without interfering with Python's +object identity system (unlike __eq__). Use this for functional comparisons +like checking if parameters have the same configuration. + +Args: + other: The object to compare with. + +Returns: + True if the objects are equivalent, False otherwise. +""" + + +class _Unset: + def __repr__(self) -> str: + return "" + + def __reduce__(self) -> tuple: + """Custom pickle support to maintain singleton pattern across contexts.""" + return (_get_unset_singleton, ()) + + +_UNSET = _Unset() + + +def _get_unset_singleton() -> _Unset: + """Return the global _UNSET singleton. + + This function is used by _Unset.__reduce__ to ensure the singleton + pattern is maintained when pickling and unpickling. + """ + return _UNSET + + +def _reconstruct_pipeline_space(attrs: Mapping[str, Any]) -> PipelineSpace: + """Reconstruct a PipelineSpace from its attributes. + + This function is used by __reduce__ to enable pickling of PipelineSpace + instances across different module contexts. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new PipelineSpace instance with the specified attributes. + """ + space = PipelineSpace() + for name, value in attrs.items(): + setattr(space, name, value) + return space + + +def _reconstruct_pipeline_space_with_class( + module_name: str, qualname: str, attrs: Mapping[str, Any] +) -> PipelineSpace: + """Reconstruct a PipelineSpace instance preserving the original class. + + This tries to import the class by module and qualname and create an instance + without calling its __init__ (to avoid side effects). If anything goes wrong + we fall back to the legacy reconstruction which returns a plain + `PipelineSpace` instance with the saved attributes. + """ + logger = logging.getLogger(__name__) + try: + module_obj = importlib.import_module(module_name) + cls_obj: Any = module_obj + for part in qualname.split("."): + cls_obj = getattr(cls_obj, part) + + if not isinstance(cls_obj, type): + raise TypeError(f"Resolved qualname is not a class: {qualname}") + + # Create instance without running __init__ so we don't require constructor args + instance: PipelineSpace = object.__new__(cls_obj) + for name, value in attrs.items(): + setattr(instance, name, value) + return instance + except (ImportError, AttributeError, TypeError, OSError) as e: + # Best-effort: restore to a plain PipelineSpace with attributes + logger.debug( + "Could not reconstruct PipelineSpace with class %s.%s: %s", + module_name, + qualname, + e, + ) + return _reconstruct_pipeline_space(attrs) + + +def _parameters_are_equivalent(param1: Any, param2: Any) -> bool: + """Check if two parameters are equivalent using their is_equivalent_to method. + + This helper function provides a safe way to compare parameters without + interfering with Python's object identity system. Falls back to regular + equality comparison for objects that don't have the is_equivalent_to method. + + Args: + param1: First parameter to compare. + param2: Second parameter to compare. + + Returns: + True if the parameters are equivalent, False otherwise. + """ + # Try to use the is_equivalent_to method if available + if hasattr(param1, "is_equivalent_to"): + return param1.is_equivalent_to(param2) + if hasattr(param2, "is_equivalent_to"): + return param2.is_equivalent_to(param1) + # Fall back to regular equality for other types + return param1 == param2 + + +@runtime_checkable +class Resolvable(Protocol): + """A protocol for objects that can be resolved into attributes.""" + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the resolvable object as a mapping.""" + raise NotImplementedError() + + def from_attrs(self, attrs: Mapping[str, Any]) -> Resolvable: + """Create a new resolvable object from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new resolvable object with the specified attributes. + """ + raise NotImplementedError() + + +def resolvable_is_fully_resolved(resolvable: Resolvable) -> bool: + """Check if a resolvable object is fully resolved. + A resolvable object is considered fully resolved if all its attributes are either + not instances of Resolvable or are themselves fully resolved. + + Args: + resolvable: Resolvable: + + Returns: + bool: True if the resolvable object is fully resolved, False otherwise. + """ + attr_objects = resolvable.get_attrs().values() + return all( + not isinstance(obj, Resolvable) or resolvable_is_fully_resolved(obj) + for obj in attr_objects + ) + + +class Fidelity(Resolvable, Generic[T]): + """A class representing a fidelity in a NePS space. + + Attributes: + domain: The domain of the fidelity, which can be an Integer or Float domain. + """ + + def __init__(self, domain: Integer | Float): + """Initialize the Fidelity with a domain. + + Args: + domain: The domain of the fidelity, which can be an Integer or Float domain. + + """ + if domain.has_prior: + raise ValueError( + "The domain of a Fidelity can not have priors, has prior:" + f" {domain.prior!r}." + ) + self.domain = domain + + def __str__(self) -> str: + """Get a string representation of the fidelity.""" + return f"Fidelity({self.domain.__str__()})" + + def compare_domain_to(self, other: object) -> bool: # noqa: D102 + # Docstring set dynamically below + if not isinstance(other, Fidelity): + return False + return self.domain == other.domain + + @property + def lower(self) -> int | float: + """Get the minimum value of the fidelity domain. + + Returns: + The minimum value of the fidelity domain. + """ + return self.domain.lower + + @property + def upper(self) -> int | float: + """Get the maximum value of the fidelity domain. + + Returns: + The maximum value of the fidelity domain. + """ + return self.domain.upper + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the fidelity as a mapping. + This method collects all attributes of the fidelity class and instance, + excluding private attributes and methods, and returns them as a dictionary. + + Returns: + A mapping of attribute names to their values. + + Raises: + ValueError: If the fidelity has no domain defined. + + """ + raise ValueError("For a Fidelity object there is nothing to resolve.") + + def from_attrs(self, attrs: Mapping[str, Any]) -> Fidelity: # noqa: ARG002 + """Create a new Fidelity instance from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new Fidelity instance with the specified attributes. + + Raises: + ValueError: If the fidelity has no domain defined. + + """ + raise ValueError("For a Fidelity object there is nothing to resolve.") + + +class IntegerFidelity(Fidelity): + """A convenience class for creating integer-valued fidelity parameters. + + This class provides a simpler interface for defining integer fidelities without + needing to explicitly wrap an Integer domain in a Fidelity. + + Example: + ```python + # Instead of: epochs = neps.Fidelity(neps.Integer(1, 50)) + # You can write: + epochs = neps.IntegerFidelity(lower=1, upper=50) + ``` + """ + + def __init__( + self, + lower: int, + upper: int, + *, + log: bool = False, + ): + """Initialize an IntegerFidelity with lower and upper bounds. + + Args: + lower: The minimum value for the integer fidelity. + upper: The maximum value for the integer fidelity. + log: Whether to sample the integer on a logarithmic scale. + + """ + super().__init__(domain=Integer(lower=lower, upper=upper, log=log)) + + def __str__(self) -> str: + """Get a string representation of the integer fidelity.""" + domain = self.domain + if domain._log: + return f"IntegerFidelity({domain.lower}, {domain.upper}, log)" + return f"IntegerFidelity({domain.lower}, {domain.upper})" + + +class FloatFidelity(Fidelity): + """A convenience class for creating float-valued fidelity parameters. + + This class provides a simpler interface for defining float fidelities without + needing to explicitly wrap a Float domain in a Fidelity. + + Example: + ```python + # Instead of: subset_ratio = neps.Fidelity(neps.Float(0.1, 1.0)) + # You can write: + subset_ratio = neps.FloatFidelity(lower=0.1, upper=1.0) + ``` + """ + + def __init__( + self, + lower: float, + upper: float, + *, + log: bool = False, + ): + """Initialize a FloatFidelity with lower and upper bounds. + + Args: + lower: The minimum value for the float fidelity. + upper: The maximum value for the float fidelity. + log: Whether to sample the float on a logarithmic scale. + + """ + super().__init__(domain=Float(lower=lower, upper=upper, log=log)) + + def __str__(self) -> str: + """Get a string representation of the float fidelity.""" + domain = self.domain + if domain._log: + return f"FloatFidelity({domain.lower}, {domain.upper}, log)" + return f"FloatFidelity({domain.lower}, {domain.upper})" + + +class PipelineSpace(Resolvable): + """A class representing a pipeline in NePS spaces.""" + + def __reduce__(self) -> tuple: + """Custom pickle support to make PipelineSpace serializable across contexts. + + This method enables PipelineSpace instances (including custom subclasses) + to be pickled and unpickled even when the original class definition is not + available (e.g., when defined in __main__ or a notebook). + + Returns: + A tuple (callable, args) for reconstructing the object. + """ + # Store the attributes and the original class identity so we can + # reconstruct an instance of the original class on unpickle. + attrs = dict(self.get_attrs()) + module_name = self.__class__.__module__ + qualname = self.__class__.__qualname__ + return (_reconstruct_pipeline_space_with_class, (module_name, qualname, attrs)) + + @property + def fidelity_attrs(self) -> Mapping[str, Fidelity]: + """Get the fidelity attributes of the pipeline. Fidelity attributes are special + attributes that represent the fidelity of the pipeline. + + Returns: + A mapping of attribute names to Fidelity objects. + """ + return {k: v for k, v in self.get_attrs().items() if isinstance(v, Fidelity)} + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the pipeline as a mapping. + This method collects all attributes of the pipeline class and instance, + excluding private attributes and methods, and returns them as a dictionary. + + Returns: + A mapping of attribute names to their values. + """ + attrs = {} + + for attr_name, attr_value in vars(self.__class__).items(): + if attr_name.startswith("_") or callable(attr_value) or attr_value is None: + continue + attrs[attr_name] = attr_value + + for attr_name, attr_value in vars(self).items(): + if attr_name.startswith("_") or callable(attr_value) or attr_value is None: + continue + attrs[attr_name] = attr_value + + properties_to_ignore = ("fidelity_attrs",) + for property_to_ignore in properties_to_ignore: + attrs.pop(property_to_ignore, None) + + return attrs + + def has_priors(self) -> bool: + """Check if any parameter in the pipeline has priors defined. + + Returns: + True if any parameter has priors, False otherwise. + """ + for param in self.get_attrs().values(): + if hasattr(param, "has_prior") and param.has_prior: + return True + return False + + def from_attrs(self, attrs: Mapping[str, Any]) -> PipelineSpace: + """Create a new Pipeline instance from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + + Returns: + A new Pipeline instance with the specified attributes. + + Raises: + ValueError: If the attributes do not match the pipeline's expected structure. + """ + new_pipeline = PipelineSpace() + for name, value in attrs.items(): + setattr(new_pipeline, name, value) + return new_pipeline + + def __str__(self) -> str: + """Get a string representation of the pipeline. + + Returns: + A string representation of the pipeline, including its class name and + attributes. + """ + from neps.space.neps_spaces.string_formatter import format_value + + # Delegate to the unified formatter + return format_value(self) + + def sample( + self, fidelity_values: dict[str, Any] | None = None + ) -> tuple[dict[str, Any], dict[str, Any]]: + """Sample a random configuration from the pipeline. + + Returns: + A tuple containing two dictionaries: + - The first dictionary is the neps-compatible config. + - The second dictionary contains the sampled values. + """ + from neps.space.neps_spaces.neps_space import ( + NepsCompatConverter, + convert_operation_to_callable, + resolve, + ) + from neps.space.neps_spaces.sampling import RandomSampler + from neps.space.neps_spaces.string_formatter import format_value + + if fidelity_values is None: + fidelity_values = {} + for name, value in self.fidelity_attrs.items(): + if name in fidelity_values: + continue + fidelity_values[name] = value.domain.sample() + + sampler = RandomSampler({}) + resolved_pipeline, resolution_context = resolve( + pipeline=self, + domain_sampler=sampler, + environment_values=fidelity_values, + ) + pipeline_dict = dict(**resolved_pipeline.get_attrs()) + + for name, value in pipeline_dict.items(): + if isinstance(value, Operation): # type: ignore[unreachable] + if isinstance(value.operator, str): # type: ignore[unreachable] + pipeline_dict[name] = format_value(value) + else: + pipeline_dict[name] = convert_operation_to_callable(value) + + return dict(NepsCompatConverter.to_neps_config(resolution_context)), pipeline_dict + + def add( + self, + new_param: Integer | Float | Categorical | Operation | Resample | Repeated, + name: str | None = None, + ) -> PipelineSpace: + """Add a new parameter to the pipeline. + + Args: + new_param: The parameter to be added, which can be an Integer, Float, + Categorical, Operation, Resample, Repeated, or PipelineSpace. + name: The name of the parameter to be added. If None, a default name will be + generated. + + Returns: + A new PipelineSpace instance with the added parameter. + + Raises: + ValueError: If the parameter is not of a supported type or if a parameter + with the same name already exists in the pipeline. + """ + if isinstance(new_param, PipelineSpace): + new_space = self + for exist_name, value in new_param.get_attrs().items(): + new_space = new_space.add(value, exist_name) + return new_space + + if not isinstance( + new_param, Integer | Float | Categorical | Operation | Resample | Repeated + ): + raise ValueError( + "Can only add Integer, Float, Categorical, Operation, Resample," + f" Repeated or PipelineSpace, got {new_param!r}." + ) + param_name = name if name else f"param_{len(self.get_attrs()) + 1}" + + # Create a new class dynamically with the added parameter + # Get all existing attributes plus the new one + new_attrs = {} + for exist_name, value in self.get_attrs().items(): + if exist_name == param_name and not _parameters_are_equivalent( + value, new_param + ): + raise ValueError( + f"A different parameter with the name {param_name!r} already exists" + " in the pipeline:\n" + f" {value}\n" + f" {new_param}" + ) + new_attrs[exist_name] = value + + # Add the new parameter if it doesn't exist + if param_name not in new_attrs: + new_attrs[param_name] = new_param + + # Create a new class with a unique name that can be pickled + new_class_name = f"{self.__class__.__name__}_added_{param_name}" + NewSpace = type(new_class_name, (self.__class__.__bases__[0],), new_attrs) + + # Set module and qualname to match original for proper pickling + NewSpace.__module__ = self.__class__.__module__ + + # Register the new class in the module's namespace so pickle can find it + import sys + + module = sys.modules[NewSpace.__module__] + setattr(module, new_class_name, NewSpace) + + return cast("PipelineSpace", NewSpace()) + + def remove(self, name: str) -> PipelineSpace: + """Remove a parameter from the pipeline by its name. This is NOT an in-place + operation. + + Args: + name: The name of the parameter to be removed. + + Returns: + A NEW PipelineSpace without the removed parameter. + + Raises: + ValueError: If no parameter with the specified name exists in the pipeline. + """ + if name not in self.get_attrs(): + raise ValueError( + f"No parameter with the name {name!r} exists in the pipeline." + ) + + # Create a new class dynamically without the removed parameter + # Get all attributes except the one to remove + new_attrs = {} + for attr_name, attr_value in self.get_attrs().items(): + if attr_name != name: + new_attrs[attr_name] = attr_value + + # Create a new class with a unique name that can be pickled + # We use the original class as the base to maintain the class hierarchy + new_class_name = f"{self.__class__.__name__}_removed_{name}" + NewSpace = type(new_class_name, (self.__class__.__bases__[0],), new_attrs) + + # Set module and qualname to match original for proper pickling + NewSpace.__module__ = self.__class__.__module__ + + # Register the new class in the module's namespace so pickle can find it + import sys + + module = sys.modules[NewSpace.__module__] + setattr(module, new_class_name, NewSpace) + + return cast("PipelineSpace", NewSpace()) + + def add_prior( + self, + parameter_name: str, + prior: Any, + prior_confidence: ConfidenceLevel | Literal["low", "medium", "high"], + ) -> PipelineSpace: + """Add a prior to a parameter in the pipeline. This is NOT an in-place operation. + + Args: + parameter_name: The name of the parameter to which the prior will be added. + prior: The value of the prior to be added. + prior_confidence: The confidence level of the prior, which can be "low", + "medium", or "high". + + Returns: + A NEW PipelineSpace with the added prior. + + Raises: + ValueError: If no parameter with the specified name exists in the pipeline + or if the parameter type does not support priors. + """ + if parameter_name not in self.get_attrs(): + raise ValueError( + f"No parameter with the name {parameter_name!r} exists in the pipeline." + ) + + # Create a new class dynamically with the modified parameter + new_attrs = {} + for exist_name, value in self.get_attrs().items(): + if exist_name == parameter_name: + if isinstance(value, Integer | Float | Categorical): + if value.has_prior: + raise ValueError( + f"The parameter {parameter_name!r} already has a prior:" + f" {value.prior!r}." + ) + if isinstance(prior_confidence, str): + prior_confidence = convert_confidence_level(prior_confidence) + old_attributes = dict(value.get_attrs()) + old_attributes["prior"] = prior + old_attributes["prior_confidence"] = prior_confidence + new_value = value.from_attrs(attrs=old_attributes) + else: + raise ValueError( + f"The parameter {parameter_name!r} is of type" + f" {type(value).__name__}, which does not support priors." + ) + else: + new_value = value + new_attrs[exist_name] = new_value + + # Create a new class with a unique name that can be pickled + new_class_name = f"{self.__class__.__name__}_prior_{parameter_name}" + NewSpace = type(new_class_name, (self.__class__.__bases__[0],), new_attrs) + + # Set module and qualname to match original for proper pickling + NewSpace.__module__ = self.__class__.__module__ + + # Register the new class in the module's namespace so pickle can find it + import sys + + module = sys.modules[NewSpace.__module__] + setattr(module, new_class_name, NewSpace) + + return cast("PipelineSpace", NewSpace()) + + +class ConfidenceLevel(enum.Enum): + """Enum representing confidence levels for sampling.""" + + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + + +def convert_confidence_level(confidence: str) -> ConfidenceLevel: + """Convert a string representation of confidence level to ConfidenceLevel enum. + + Args: + confidence: A string representing the confidence level, e.g., "low", "medium", + "high". + + Returns: + ConfidenceLevel: The corresponding ConfidenceLevel enum value. + + Raises: + ValueError: If the input string does not match any of the defined confidence + levels. + """ + try: + return ConfidenceLevel[confidence.upper()] + except KeyError as e: + raise ValueError(f"Invalid confidence level: {confidence}") from e + + +class Domain(Resolvable, abc.ABC, Generic[T]): + """An abstract base class representing a domain in NePS spaces.""" + + @property + @abc.abstractmethod + def lower(self) -> T: + """Get the minimum value of the domain.""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def upper(self) -> T: + """Get the maximum value of the domain.""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def has_prior(self) -> bool: + """Check if the domain has a prior defined.""" + raise NotImplementedError() + + @property + @abc.abstractmethod + def prior(self) -> T: + """Get the prior value of the domain. + Raises ValueError if the domain has no prior defined. + + """ + raise NotImplementedError() + + @property + @abc.abstractmethod + def prior_confidence(self) -> ConfidenceLevel: + """Get the confidence level of the prior. + Raises ValueError if the domain has no prior defined. + + """ + raise NotImplementedError() + + @property + @abc.abstractmethod + def range_compatibility_identifier(self) -> str: + """Get a string identifier for the range compatibility of the domain. + This identifier is used to check if two domains are compatible based on their + ranges. + + """ + raise NotImplementedError() + + @abc.abstractmethod + def sample(self) -> T: + """Sample a value from the domain. + Returns a value of type T that is within the domain's range. + + """ + raise NotImplementedError() + + @abc.abstractmethod + def centered_around( + self, + center: T, + confidence: ConfidenceLevel, + ) -> Domain[T]: + """Create a new domain centered around a given value with a specified confidence + level. + + Args: + center: The value around which to center the new domain. + confidence: The confidence level for the new domain. + center: T: + confidence: ConfidenceLevel: + + Returns: + A new Domain instance that is centered around the specified value. + + Raises: + ValueError: If the center value is not within the domain's range. + + """ + raise NotImplementedError() + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the domain as a mapping. + This method collects all attributes of the domain class and instance, + excluding private attributes and methods, and returns them as a dictionary. + + Returns: + A mapping of attribute names to their values. + """ + return {k.lstrip("_"): v for k, v in vars(self).items()} + + def from_attrs(self, attrs: Mapping[str, Any]) -> Domain[T]: + """Create a new Domain instance from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new Domain instance with the specified attributes. + + Raises: + ValueError: If the attributes do not match the domain's expected structure. + + """ + return type(self)(**attrs) + + def resample(self) -> Resample: + """Wrap this domain in a Resample container. + + This allows resampling the domain each time it's resolved, useful for + creating dynamic structures where the same parameter definition is + sampled multiple times independently. + + Returns: + A Resample instance wrapping this domain. + + Example: + ```python + # Instead of: neps.Resample(neps.Integer(1, 10)) + # You can write: neps.Integer(1, 10).resample() + ``` + """ + return Resample(self) + + +def _calculate_new_domain_bounds( + number_type: type[int] | type[float], + lower: int | float, + upper: int | float, + center: int | float, + confidence: ConfidenceLevel, +) -> tuple[int, int] | tuple[float, float]: + """Calculate new bounds for a domain based on a center value and confidence level. + This function determines the new minimum and maximum values for a domain based on + a given center value and a confidence level. It splits the domain range into chunks + and adjusts the bounds based on the specified confidence level. + + Args: + number_type: The type of numbers in the domain (int or float). + lower: The minimum value of the domain. + upper: The maximum value of the domain. + center: The center value around which to calculate the new bounds. + confidence: The confidence level for the new bounds. + + Returns: + A tuple containing the new minimum and maximum values for the domain. + + Raises: + ValueError: If the center value is not within the domain's range or if the + number_type is not supported. + """ + if center < lower or center > upper: + raise ValueError( + f"Center value {center!r} must be within domain range [{lower!r}, {upper!r}]" + ) + + # Determine a chunk size by splitting the domain range into a fixed number of chunks. + # Then use the confidence level to decide how many chunks to include + # around the given center (on each side). + + number_of_chunks = 10.0 + chunk_size = (upper - lower) / number_of_chunks + + # The numbers refer to how many segments to have on each side of the center. + # TODO: [lum] we need to make sure that in the end the range does not just have the + # center, but at least a little bit more around it too. + confidence_to_number_of_chunks_on_each_side = { + ConfidenceLevel.HIGH: 1.0, + ConfidenceLevel.MEDIUM: 2.5, + ConfidenceLevel.LOW: 4.0, + } + + chunk_multiplier = confidence_to_number_of_chunks_on_each_side[confidence] + interval_radius = chunk_size * chunk_multiplier + + if number_type is int: + # In this case we need to use ceil/floor so that we end up with ints. + new_min = max(lower, math.floor(center - interval_radius)) + new_max = min(upper, math.ceil(center + interval_radius)) + elif number_type is float: + new_min = max(lower, center - interval_radius) + new_max = min(upper, center + interval_radius) + else: + raise ValueError(f"Unsupported number type {number_type!r}.") + + return new_min, new_max + + +class Categorical(Domain[int], Generic[T]): + """A domain representing a categorical choice from a set of options. + + Attributes: + choices: A tuple of choices or a Domain of choices. + prior: The index of the prior choice in the choices tuple. + prior_confidence: The confidence level of the prior choice. + """ + + def __init__( + self, + choices: ( + tuple[T | Domain[T] | Resolvable | Any, ...] + | Sequence[T | Domain[T] | Resolvable | Any] + | Domain[T] + | Resolvable + ), + prior: int | Domain[int] | _Unset = _UNSET, + prior_confidence: ( + ConfidenceLevel | Literal["low", "medium", "high"] | _Unset + ) = _UNSET, + ): + """Initialize the Categorical domain with choices and optional prior. + + Args: + choices: A tuple or list of choices or a Domain of choices. + prior: The index of the prior choice in the choices tuple. + prior_confidence: The confidence level of the prior choice. + + """ + self._choices: ( + tuple[T | Domain[T] | Resolvable | Any, ...] + | Sequence[T | Domain[T] | Resolvable | Any] + | Domain[T] + ) + if isinstance(choices, Sequence): + self._choices = tuple(choice for choice in choices) + if any(isinstance(choice, tuple) for choice in self._choices) and any( + not isinstance(choice, tuple) for choice in self._choices + ): + self._choices = tuple( + (choice,) if not isinstance(choice, tuple) else choice + for choice in self._choices + ) + else: + self._choices = choices # type: ignore[assignment] + self._prior = prior + self._prior_confidence = ( + convert_confidence_level(prior_confidence) + if isinstance(prior_confidence, str) + else prior_confidence + ) + if self._prior is not _UNSET and self._prior_confidence is _UNSET: + raise ValueError( + "If prior is set, prior_confidence must also be set to a valid value." + ) + + def __str__(self) -> str: + """Get a string representation of the categorical domain.""" + from neps.space.neps_spaces.string_formatter import format_value + + return format_value(self) + + def compare_domain_to(self, other: object) -> bool: # noqa: D102 + # Docstring set dynamically below + if not isinstance(other, Categorical): + return False + return ( + self._prior == other.prior + and self._prior_confidence == other.prior_confidence + and self.choices == other.choices + ) + + @property + def lower(self) -> int: + """Get the minimum value of the categorical domain. + + Returns: + The minimum index of the choices, which is always 0. + + """ + return 0 + + @property + def upper(self) -> int: + """Get the maximum value of the categorical domain. + + Returns: + The maximum index of the choices, which is the length of the choices tuple + minus one. + + """ + return max(len(cast("tuple", self._choices)) - 1, 0) + + @property + def choices(self) -> tuple[T | Domain[T] | Resolvable, ...] | Domain[T]: + """Get the choices available in the categorical domain. + + Returns: + A tuple of choices or a Domain of choices. + + """ + return ( + self._choices + if not isinstance(self._choices, Sequence) + else tuple(self._choices) + ) + + @property + def has_prior(self) -> bool: + """Check if the categorical domain has a prior defined. + + Returns: + True if the prior and prior confidence are set, False otherwise. + """ + return self._prior is not _UNSET and self._prior_confidence is not _UNSET + + @property + def prior(self) -> int: + """Get the prior index of the categorical domain. + + Returns: + The index of the prior choice in the choices tuple. + + Raises: + ValueError: If the domain has no prior defined. + + """ + if not self.has_prior: + raise ValueError("Domain has no prior and prior_confidence defined.") + return int(cast("int", self._prior)) + + @property + def prior_confidence(self) -> ConfidenceLevel: + """Get the confidence level of the prior choice. + + Returns: + The confidence level of the prior choice. + + Raises: + ValueError: If the domain has no prior defined. + + """ + if not self.has_prior: + raise ValueError("Domain has no prior and prior_confidence defined.") + return cast("ConfidenceLevel", self._prior_confidence) + + @property + def range_compatibility_identifier(self) -> str: + """Get a string identifier for the range compatibility of the categorical domain. + + Returns: + A string representation of the number of choices in the domain. + + """ + return f"{len(cast('tuple', self._choices))}" + + def sample(self) -> int: + """Sample a random index from the categorical choices. + + Returns: + A randomly selected index from the choices tuple. + + Raises: + ValueError: If the choices are empty. + + """ + return int(random.randint(0, len(cast("tuple[T]", self._choices)) - 1)) + + def centered_around( + self, + center: int, + confidence: ConfidenceLevel, + ) -> Categorical: + """Create a new categorical domain centered around a specific choice index. + + Args: + center: The index of the choice around which to center the new domain. + confidence: The confidence level for the new domain. + center: int: + confidence: ConfidenceLevel: + + Returns: + A new Categorical instance with a range centered around the specified + choice index. + + Raises: + ValueError: If the center index is out of bounds of the choices. + + """ + new_min, new_max = cast( + "tuple[int, int]", + _calculate_new_domain_bounds( + number_type=int, + lower=self.lower, + upper=self.upper, + center=center, + confidence=confidence, + ), + ) + new_choices = cast("tuple", self._choices)[new_min : new_max + 1] + return Categorical( + choices=new_choices, + prior=new_choices.index(cast("tuple", self._choices)[center]), + prior_confidence=confidence, + ) + + +class Float(Domain[float]): + """A domain representing a continuous range of floating-point values. + + Attributes: + lower: The minimum value of the domain. + upper: The maximum value of the domain. + log: Whether to sample values on a logarithmic scale. + prior: The prior value for the domain, if any. + prior_confidence: The confidence level of the prior value. + """ + + def __init__( + self, + lower: float, + upper: float, + log: bool = False, # noqa: FBT001, FBT002 + prior: float | _Unset = _UNSET, + prior_confidence: ( + Literal["low", "medium", "high"] | ConfidenceLevel | _Unset + ) = _UNSET, + **kwargs: object, + ): + """Initialize the Float domain with min and max values, and optional prior. + + Args: + lower: The minimum value of the domain. + upper: The maximum value of the domain. + log: Whether to sample values on a logarithmic scale. + prior: The prior value for the domain, if any. + prior_confidence: The confidence level of the prior value. + **kwargs: Additional keyword arguments (e.g., is_fidelity) are accepted + for backward compatibility but ignored in PipelineSpace parameters. + + """ + # Handle is_fidelity for backward compatibility + # Store it silently - user will get warning from neps.run about SearchSpace usage + # TODO: Remove this when removing SearchSpace support + if "is_fidelity" in kwargs: + warnings.warn( + "`is_fidelity` argument is deprecated and will be removed in future" + " versions. Please update your code accordingly.", + DeprecationWarning, + stacklevel=2, + ) + self._is_fidelity_compat = bool(kwargs.get("is_fidelity", False)) + if any(key != "is_fidelity" for key in kwargs): + raise TypeError(f"Unexpected keyword arguments: {', '.join(kwargs.keys())}.") + + self._lower = lower + self._upper = upper + self._log = log + self._prior = prior + self._prior_confidence = ( + convert_confidence_level(prior_confidence) + if isinstance(prior_confidence, str) + else prior_confidence + ) + if self._prior is not _UNSET and self._prior_confidence is _UNSET: + raise ValueError( + "If prior is set, prior_confidence must also be set to a valid value." + ) + + def __str__(self) -> str: + """Get a string representation of the floating-point domain.""" + from neps.space.neps_spaces.string_formatter import format_value + + return format_value(self) + + def compare_domain_to(self, other: object) -> bool: # noqa: D102 + # Docstring set dynamically below + if not isinstance(other, Float): + return False + return ( + self._prior == other.prior + and self._prior_confidence == other.prior_confidence + and self.lower == other.lower + and self.upper == other.upper + and self._log == other.log + ) + + @property + def lower(self) -> float: + """Get the minimum value of the floating-point domain. + + Returns: + The minimum value of the domain. + + Raises: + ValueError: If lower is greater than upper. + + """ + return self._lower + + @property + def upper(self) -> float: + """Get the maximum value of the floating-point domain. + + Returns: + The maximum value of the domain. + + Raises: + ValueError: If lower is greater than upper. + + """ + return self._upper + + @property + def log(self) -> bool: + """Check if the floating-point domain uses logarithmic sampling. + + Returns: + True if values should be sampled on a logarithmic scale, False otherwise. + + """ + return self._log + + @property + def has_prior(self) -> bool: + """Check if the floating-point domain has a prior defined. + + Returns: + True if the prior and prior confidence are set, False otherwise. + + """ + return self._prior is not _UNSET and self._prior_confidence is not _UNSET + + @property + def prior(self) -> float: + """Get the prior value of the floating-point domain. + + Returns: + The prior value of the domain. + + Raises: + ValueError: If the domain has no prior defined. + + """ + if not self.has_prior: + raise ValueError("Domain has no prior and prior_confidence defined.") + return float(cast("float", self._prior)) + + @property + def prior_confidence(self) -> ConfidenceLevel: + """Get the confidence level of the prior value. + + Returns: + The confidence level of the prior value. + + Raises: + ValueError: If the domain has no prior defined. + + """ + if not self.has_prior: + raise ValueError("Domain has no prior and prior_confidence defined.") + return cast("ConfidenceLevel", self._prior_confidence) + + @property + def range_compatibility_identifier(self) -> str: + """Get a string identifier for the range compatibility of the floating-point + domain. + + Returns: + A string representation of the minimum and maximum values, and whether + the domain is logarithmic. + + """ + return f"{self._lower}_{self._upper}_{self._log}" + + def sample(self) -> float: + """Sample a random floating-point value from the domain. + + Returns: + A randomly selected floating-point value within the domain's range. + + Raises: + ValueError: If lower is greater than upper. + + """ + if self._log: + log_min = math.log(self._lower) + log_max = math.log(self._upper) + return float(math.exp(random.uniform(log_min, log_max))) + return float(random.uniform(self._lower, self._upper)) + + def centered_around( + self, + center: float, + confidence: ConfidenceLevel, + ) -> Float: + """Create a new floating-point domain centered around a specific value. + + Args: + center: The value around which to center the new domain. + confidence: The confidence level for the new domain. + center: float: + confidence: ConfidenceLevel: + + Returns: + A new Float instance that is centered around the specified value. + + Raises: + ValueError: If the center value is not within the domain's range. + + """ + new_min, new_max = _calculate_new_domain_bounds( + number_type=float, + lower=self.lower, + upper=self.upper, + center=center, + confidence=confidence, + ) + return Float( + lower=new_min, + upper=new_max, + log=self._log, + prior=center, + prior_confidence=confidence, + ) + + +class Integer(Domain[int]): + """A domain representing a range of integer values. + + Attributes: + lower: The minimum value of the domain. + upper: The maximum value of the domain. + log: Whether to sample values on a logarithmic scale. + prior: The prior value for the domain, if any. + prior_confidence: The confidence level of the prior value. + """ + + def __init__( + self, + lower: int, + upper: int, + log: bool = False, # noqa: FBT001, FBT002 + prior: float | int | _Unset = _UNSET, + prior_confidence: ( + Literal["low", "medium", "high"] | ConfidenceLevel | _Unset + ) = _UNSET, + **kwargs: object, + ): + """Initialize the Integer domain with min and max values, and optional prior. + + Args: + lower: The minimum value of the domain. + upper: The maximum value of the domain. + log: Whether to sample values on a logarithmic scale. + prior: The prior value for the domain, if any. + prior_confidence: The confidence level of the prior value. + **kwargs: Additional keyword arguments (e.g., is_fidelity) are accepted + for backward compatibility but ignored in PipelineSpace parameters. + """ + # Handle is_fidelity for backward compatibility + # Store it silently - user will get warning from neps.run about SearchSpace usage + # TODO: Remove this when removing SearchSpace support + if "is_fidelity" in kwargs: + warnings.warn( + "`is_fidelity` argument is deprecated and will be removed in future" + " versions. Please update your code accordingly.", + DeprecationWarning, + stacklevel=2, + ) + self._is_fidelity_compat = bool(kwargs.get("is_fidelity", False)) + if any(key != "is_fidelity" for key in kwargs): + raise TypeError(f"Unexpected keyword arguments: {', '.join(kwargs.keys())}.") + + self._lower = lower + self._upper = upper + self._log = log + self._prior = prior + self._prior_confidence = ( + convert_confidence_level(prior_confidence) + if isinstance(prior_confidence, str) + else prior_confidence + ) + if self._prior != _UNSET and self._prior_confidence is _UNSET: + raise ValueError( + "If prior is set, prior_confidence must also be set to a valid value." + ) + + def __str__(self) -> str: + """Get a string representation of the integer domain.""" + from neps.space.neps_spaces.string_formatter import format_value + + return format_value(self) + + def compare_domain_to(self, other: object) -> bool: # noqa: D102 + # Docstring set dynamically below + if not isinstance(other, Integer): + return False + return ( + self._prior == other.prior + and self._prior_confidence == other.prior_confidence + and self.lower == other.lower + and self.upper == other.upper + and self._log == other.log + ) + + @property + def lower(self) -> int: + """Get the minimum value of the integer domain. + + Returns: + The minimum value of the domain. + + Raises: + ValueError: If lower is greater than upper. + + """ + return self._lower + + @property + def upper(self) -> int: + """Get the maximum value of the integer domain. + + Returns: + The maximum value of the domain. + + Raises: + ValueError: If lower is greater than upper. + + """ + return self._upper + + @property + def log(self) -> bool: + """Check if the integer domain uses logarithmic sampling. + + Returns: + True if values should be sampled on a logarithmic scale, False otherwise. + + """ + return self._log + + @property + def has_prior(self) -> bool: + """Check if the integer domain has a prior defined. + + Returns: + True if the prior and prior confidence are set, False otherwise. + + """ + return self._prior is not _UNSET and self._prior_confidence is not _UNSET + + @property + def prior(self) -> int: + """Get the prior value of the integer domain. + + Returns: + The prior value of the domain. + + Raises: + ValueError: If the domain has no prior defined. + + """ + if not self.has_prior: + raise ValueError("Domain has no prior and prior_confidence defined.") + return int(cast("int", self._prior)) + + @property + def prior_confidence(self) -> ConfidenceLevel: + """Get the confidence level of the prior value. + + Returns: + The confidence level of the prior value. + + Raises: + ValueError: If the domain has no prior defined. + + """ + if not self.has_prior: + raise ValueError("Domain has no prior and prior_confidence defined.") + return cast("ConfidenceLevel", self._prior_confidence) + + @property + def range_compatibility_identifier(self) -> str: + """Get a string identifier for the range compatibility of the integer domain. + + Returns: + A string representation of the minimum and maximum values, and whether + the domain is logarithmic. + + """ + return f"{self._lower}_{self._upper}_{self._log}" + + def sample(self) -> int: + """Sample a random integer value from the domain. + + Returns: + A randomly selected integer value within the domain's range. + + """ + if self._log: + return int( + math.exp(random.uniform(math.log(self._lower), math.log(self._upper))) + ) + return int(random.randint(self._lower, self._upper)) + + def centered_around( + self, + center: int, + confidence: ConfidenceLevel, + ) -> Integer: + """Create a new integer domain centered around a specific value. + + Args: + center: The value around which to center the new domain. + confidence: The confidence level for the new domain. + center: int: + confidence: ConfidenceLevel: + + Returns: + A new Integer instance that is centered around the specified value. + + Raises: + ValueError: If the center value is not within the domain's range. + + """ + new_min, new_max = cast( + "tuple[int, int]", + _calculate_new_domain_bounds( + number_type=int, + lower=self.lower, + upper=self.upper, + center=center, + confidence=confidence, + ), + ) + return Integer( + lower=new_min, + upper=new_max, + log=self._log, + prior=center, + prior_confidence=confidence, + ) + + +class Operation(Resolvable): + """A class representing an operation in a NePS space. + + Attributes: + operator: The operator to be used in the operation, can be a callable or a string. + args: A sequence of arguments to be passed to the operator. + kwargs: A mapping of keyword arguments to be passed to the operator. + """ + + def __init__( + self, + operator: Callable | str, + args: Sequence[Any] | Resolvable | None = None, + kwargs: Mapping[str, Any] | Resolvable | None = None, + ): + """Initialize the Operation with an operator, arguments, and keyword arguments. + + Args: + operator: The operator to be used in the operation, can be a callable or a + string. + args: A sequence of arguments to be passed to the operator. + kwargs: A mapping of keyword arguments to be passed to the operator. + + """ + self._operator = operator + + self._args: tuple[Any, ...] | Resolvable + if not isinstance(args, Resolvable): + self._args = tuple(args) if args else () + else: + self._args = args + + self._kwargs: Mapping[str, Any] | Resolvable + if not isinstance(kwargs, Resolvable): + self._kwargs = kwargs if kwargs else {} + else: + self._kwargs = kwargs + + def __str__(self) -> str: + """Get a string representation of the operation.""" + from neps.space.neps_spaces.string_formatter import format_value + + return format_value(self) + + def resample(self) -> Resample: # noqa: D102 + # Docstring set dynamically below + return Resample(self) + + def compare_domain_to(self, other: object) -> bool: # noqa: D102 + # Docstring set dynamically below + if not isinstance(other, Operation): + return False + return ( + self.operator == other.operator + and self.args == other.args + and self.kwargs == other.kwargs + ) + + @property + def operator(self) -> Callable | str: + """Get the operator of the operation. + + Returns: + The operator, which can be a callable or a string. + + Raises: + ValueError: If the operator is not callable or a string. + + """ + return self._operator + + @property + def args(self) -> tuple[Any, ...]: + """Get the arguments of the operation. + + Returns: + A tuple of arguments to be passed to the operator. + + Raises: + ValueError: If the args are not resolved to a tuple. + """ + if isinstance(self._args, Resolvable): + raise ValueError( + f"Operation args contain unresolved Resolvable: {self._args!r}. " + "The operation needs to be resolved before accessing args as a tuple." + ) + return self._args + + @property + def kwargs(self) -> Mapping[str, Any]: + """Get the keyword arguments of the operation. + + Returns: + A mapping of keyword arguments to be passed to the operator. + + Raises: + ValueError: If the kwargs are not resolved to a mapping. + """ + if isinstance(self._kwargs, Resolvable): + raise ValueError( + f"Operation kwargs contain unresolved Resolvable: {self._kwargs!r}. " + "The operation needs to be resolved before accessing kwargs as a mapping." + ) + return self._kwargs + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the operation as a mapping. + This method collects all attributes of the operation class and instance, + excluding private attributes and methods, and returns them as a dictionary. + + Returns: + A mapping of attribute names to their values. + + """ + return {k.lstrip("_"): v for k, v in vars(self).items()} + + def from_attrs(self, attrs: Mapping[str, Any]) -> Operation: + """Create a new Operation instance from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new Operation instance with the specified attributes. + + Raises: + ValueError: If the attributes do not match the operation's expected structure. + + """ + return type(self)(**attrs) + + +# TODO: [lum] For tuples, lists and dicts, +# should we make the behavior similar to other resolvables, +# in that they will be cached and then we also need to use Resample for them? + + +class ByName(Resolvable): + """A reference to another parameter by its string name. + + This class is used for self-references or forward-references in complex spaces, + providing a clearer and more consistent API than using plain strings. + + Attributes: + name: The string name of the parameter to reference. + + Example: + ```python + class RecursiveSpace(neps.PipelineSpace): + # Self-reference using ByName + self_ref = neps.Categorical( + choices=( + (neps.ByName("self_ref").resample(), + neps.ByName("self_ref").resample()), + (neps.ByName("future_param").resample(),), + ) + ) + + future_param = neps.Float(lower=0, upper=5) + ``` + """ + + def __init__(self, name: str): + """Initialize the ByName reference with a parameter name. + + Args: + name: The string name of the parameter to reference. + + Raises: + ValueError: If the name is not a valid string. + """ + if not isinstance(name, str) or not name: + raise ValueError( + f"ByName requires a non-empty string name. Received: {name!r}" + ) + self._name = name + + def __str__(self) -> str: + from neps.space.neps_spaces.string_formatter import format_value + + return format_value(self) + + @property + def name(self) -> str: + """Get the referenced parameter name. + + Returns: + The string name of the referenced parameter. + """ + return self._name + + def resample(self) -> Resample: + """Wrap this reference in a Resample container. + + This allows resampling the referenced parameter each time it's resolved, + useful for creating dynamic structures where the same parameter is sampled + multiple times independently. + + Returns: + A Resample instance wrapping this ByName reference. + + Example: + ```python + # Instead of: neps.Resample(neps.ByName("param_name")) + # You can write: neps.ByName("param_name").resample() + ``` + """ + return Resample(self) + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the ByName reference as a mapping. + + Returns: + A mapping containing the referenced parameter name. + """ + return {"name": self._name} + + def from_attrs(self, attrs: Mapping[str, Any]) -> ByName: + """Create a new ByName reference from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new ByName instance with the specified name. + """ + return ByName(name=attrs["name"]) + + def compare_domain_to(self, other: object) -> bool: + """Check if this ByName reference is equivalent to another. + + Args: + other: The object to compare with. + + Returns: + True if both are ByName references with the same name, False otherwise. + """ + if not isinstance(other, ByName): + return False + return self.name == other.name + + +class Resample(Resolvable): + """A class representing a resampling operation in a NePS space. + + Attributes: + source: The source of the resampling, which can be a resolvable object or a + string. + """ + + def __init__(self, source: Resolvable): + """Initialize the Resample object with a source. + + Args: + source: The source of the resampling, must be a resolvable object + (including ByName for parameter name references). + + Raises: + ValueError: If source is a Fidelity object or a string. + """ + if isinstance(source, Fidelity): + raise ValueError("Fidelity objects cannot be resampled.") + + if isinstance(source, str): + raise TypeError( + "Resample does not accept plain strings. To reference a parameter by" + f" name, use neps.ByName('{source}').resample() instead of" + f" neps.Resample('{source}')." + ) + + self._source = source + + def __str__(self) -> str: + from neps.space.neps_spaces.string_formatter import format_value + + return format_value(self) + + @property + def source(self) -> Resolvable: + """Get the source of the resampling. + + Returns: + The source of the resampling, which is a resolvable object (including ByName). + + """ + return self._source + + @property + def is_resampling_by_name(self) -> bool: + """Check if the resampling is by name reference. + + Returns: + True if the source is a ByName reference, indicating a resampling by name, + False if the source is another type of resolvable object. + + """ + return isinstance(self._source, ByName) + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the resampling source as a mapping. + + Returns: + A mapping of attribute names to their values. + + Raises: + ValueError: If the resampling is by name reference. + + """ + if self.is_resampling_by_name: + # ByName references delegate to their own get_attrs + return self._source.get_attrs() + return self._source.get_attrs() + + def from_attrs(self, attrs: Mapping[str, Any]) -> Resolvable: + """Create a new resolvable object from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new resolvable object created from the specified attributes. + + Raises: + ValueError: If the resampling is by name reference. + + """ + if self.is_resampling_by_name: + # ByName references delegate to their own from_attrs + return self._source.from_attrs(attrs) + return self._source.from_attrs(attrs) + + def compare_domain_to(self, other: object) -> bool: # noqa: D102 + # Docstring set dynamically below + if not isinstance(other, Resample): + return False + return self.source == other.source + + +class Repeated(Resolvable): + """A class representing a sequence where a resolvable + is repeated a variable number of times. + + Attributes: + count: The count how many times the content should be repeated. + content: The content which will be repeated. + """ + + def __init__( + self, + count: int | Domain[int] | Resolvable, + content: Resolvable | Any, + ): + """Initialize the Repeated object with a count and content. + + Args: + count: The count how many times the content should be repeated. + content: The content which will be repeated. + """ + if isinstance(count, int) and count < 0: + raise ValueError(f"The received repeat count is negative. Received {count!r}") + + self._count = count + self._content = content + + @property + def count(self) -> int | Domain[int] | Resolvable: + """Get the count how many times the content should be repeated. + + Returns: + The count how many times the content will be repeated. + """ + return self._count + + @property + def content(self) -> Resolvable | Any: + """Get the content which will be repeated. + + Returns: + The content which will be repeated. + """ + return self._content + + def resample(self) -> Resample: # noqa: D102 + # Docstring set dynamically below + return Resample(self) + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the resolvable as a mapping. + + Returns: + A mapping of attribute names to their values. + """ + return {"count": self.count, "content": self.content} + + def from_attrs(self, attrs: Mapping[str, Any]) -> Resolvable: + """Create a new resolvable object from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new resolvable object created from the specified attributes. + """ + return Repeated(count=attrs["count"], content=attrs["content"]) + + +class Lazy(Resolvable): + """A class representing a lazy operation in a NePS space. + + The purpose is to have the resolution process + stop at the moment it gets to this object, + preventing the resolution of the object it wraps. + + Attributes: + content: The content held, which can be a resolvable object or a + tuple or a string. + """ + + def __init__(self, content: Resolvable | tuple[Any] | str): + """Initialize the Lazy object with content. + + Args: + content: The content being held, which can be a resolvable object + or a tuple or a string. + """ + self._content = content + + @property + def content(self) -> Resolvable | tuple[Any] | str: + """Get the content being held. + + Returns: + The content of the lazy resolvable, which can be a resolvable object + or a tuple or a string. + """ + return self._content + + def resample(self) -> Resample: # noqa: D102 + # Docstring set dynamically below + return Resample(self) + + def get_attrs(self) -> Mapping[str, Any]: + """Get the attributes of the lazy resolvable as a mapping. + + Raises: + ValueError: Always, since this operation does not make sense here. + """ + raise ValueError( + f"This is a lazy resolvable. Can't get attrs from it: {self.content!r}." + ) + + def from_attrs(self, attrs: Mapping[str, Any]) -> Resolvable: # noqa: ARG002 + """Create a new resolvable object from the given attributes. + + Args: + attrs: A mapping of attribute names to their values. + + Returns: + A new resolvable object created from the specified attributes. + + Raises: + ValueError: Always, since this operation does not make sense here. + + + """ + raise ValueError( + f"This is a lazy resolvable. Can't create object for it: {self.content!r}." + ) + + +# TODO: [lum] all the `get_attrs` and `from_attrs` MUST NOT raise. +# They should return the best representation of themselves that they can. +# This is because all resolvable objects can be nested content of other +# resolvable objects that in general will interact with them +# through these two methods. +# When they raise, then the traversal will not be possible. + + +# Set docstrings dynamically to maintain DRY principle +# This avoids repeating identical documentation across multiple classes +Operation.resample.__doc__ = _RESAMPLE_DOCSTRING.format( + type_name="operation", + description="operation", + type_lower="operation", +) +Repeated.resample.__doc__ = _RESAMPLE_DOCSTRING.format( + type_name="repeated structure", + description="repeated structure", + type_lower="repeated", +) +Lazy.resample.__doc__ = _RESAMPLE_DOCSTRING.format( + type_name="lazy value", + description="lazy value", + type_lower="lazy", +) + +Fidelity.compare_domain_to.__doc__ = _COMPARE_DOMAIN_DOCSTRING.format( + type_name="fidelity" +) +Categorical.compare_domain_to.__doc__ = _COMPARE_DOMAIN_DOCSTRING.format( + type_name="categorical" +) +Float.compare_domain_to.__doc__ = _COMPARE_DOMAIN_DOCSTRING.format(type_name="float") +Integer.compare_domain_to.__doc__ = _COMPARE_DOMAIN_DOCSTRING.format(type_name="integer") +Operation.compare_domain_to.__doc__ = _COMPARE_DOMAIN_DOCSTRING.format( + type_name="operation" +) +Resample.compare_domain_to.__doc__ = _COMPARE_DOMAIN_DOCSTRING.format( + type_name="resampled" +) diff --git a/neps/space/neps_spaces/sampling.py b/neps/space/neps_spaces/sampling.py new file mode 100644 index 000000000..5f1136a8e --- /dev/null +++ b/neps/space/neps_spaces/sampling.py @@ -0,0 +1,658 @@ +"""This module defines various samplers for NEPS spaces, allowing for different sampling +strategies such as predefined values, random sampling, and mutation-based sampling. +""" + +from __future__ import annotations + +import random +from collections.abc import Mapping +from typing import Any, Protocol, TypeVar, cast, runtime_checkable + +from scipy import stats + +from neps.sampling.priors import PRIOR_CONFIDENCE_MAPPING +from neps.space.neps_spaces.parameters import ( + Categorical, + ConfidenceLevel, + Domain, + Float, + Integer, + PipelineSpace, +) +from neps.validation import validate_parameter_value + +T = TypeVar("T") +P = TypeVar("P", bound="PipelineSpace") + + +@runtime_checkable +class DomainSampler(Protocol): + """A protocol for domain samplers that can sample from a given domain.""" + + def __call__( + self, + *, + domain_obj: Domain[T], + current_path: str, + ) -> T: + """Sample a value from the given domain. + + Args: + domain_obj: The domain object to sample from. + current_path: The current path in the resolution context. + + Returns: + A sampled value of type T from the domain. + + Raises: + NotImplementedError: If the method is not implemented. + """ + raise NotImplementedError() + + +class OnlyPredefinedValuesSampler(DomainSampler): + """A sampler that only returns predefined values for a given path. + If the path is not found in the predefined values, it raises a ValueError. + + Args: + predefined_samplings: A mapping of paths to predefined values. + """ + + def __init__( + self, + predefined_samplings: Mapping[str, Any], + ): + """Initialize the sampler with predefined samplings. + + Args: + predefined_samplings: A mapping of paths to predefined values. + + Raises: + ValueError: If predefined_samplings is empty. + """ + self._predefined_samplings = predefined_samplings + + def __call__( + self, + *, + domain_obj: Domain[T], # noqa: ARG002 + current_path: str, + ) -> T: + """Sample a value from the predefined samplings for the given path. + + Args: + domain_obj: The domain object, not used in this sampler. + current_path: The path for which to sample a value. + + Returns: + The predefined value for the given path. + + Raises: + ValueError: If the current path is not in the predefined samplings. + """ + if current_path not in self._predefined_samplings: + raise ValueError(f"No predefined value for path: {current_path!r}.") + return cast("T", self._predefined_samplings[current_path]) + + +class RandomSampler(DomainSampler): + """A sampler that randomly samples from a predefined set of values. + If the current path is not in the predefined values, it samples from the domain. + + Args: + predefined_samplings: A mapping of paths to predefined values. + This sampler will use these values if available, otherwise it will sample + from the domain. + """ + + def __init__( + self, + predefined_samplings: Mapping[str, Any], + ): + """Initialize the sampler with predefined samplings. + + Args: + predefined_samplings: A mapping of paths to predefined values. + + Raises: + ValueError: If predefined_samplings is empty. + """ + self._predefined_samplings = predefined_samplings + + def __call__( + self, + *, + domain_obj: Domain[T], + current_path: str, + ) -> T: + """Sample a value from the predefined samplings or the domain. + + Args: + domain_obj: The domain object from which to sample. + current_path: The path for which to sample a value. + + Returns: + A sampled value, either from the predefined samplings or from the + domain. + + Raises: + ValueError: If the current path is not in the predefined samplings and + the domain does not have a prior defined. + """ + if current_path not in self._predefined_samplings: + sampled_value = domain_obj.sample() + else: + sampled_value = cast("T", self._predefined_samplings[current_path]) + return sampled_value + + +class IOSampler(DomainSampler): + """A sampler that samples by asking the user at each decision.""" + + def __call__( # noqa: C901, PLR0912 + self, + *, + domain_obj: Domain[T], + current_path: str, + ) -> T: + """Sample a value from the predefined samplings or the domain. + + Args: + domain_obj: The domain object from which to sample. + current_path: The current path in the search space. + + Returns: + A value from the user input. + """ + if isinstance(domain_obj, Float | Integer): + print( + "Please provide" + f" {'a float' if isinstance(domain_obj, Float) else 'an integer'} value" + f" for \n\t'{current_path}'\nin the range [{domain_obj.lower}," # type: ignore[attr-defined] + f" {domain_obj.upper}]: ", # type: ignore[attr-defined] + ) + elif isinstance(domain_obj, Categorical): + from neps.space.neps_spaces.string_formatter import format_value + + # Format choices and check for multi-line content + formatted_choices = [format_value(c, indent=0) for c in domain_obj.choices] # type: ignore[attr-defined, arg-type] + has_multiline = any("\n" in formatted for formatted in formatted_choices) + + # Build choices display + choices_lines = [""] if has_multiline else [] + for n, formatted in enumerate(formatted_choices): + if "\n" in formatted: + choices_lines.append(f"Option {n}:") + choices_lines.append(formatted) + else: + choices_lines.append(f"Option {n}: {formatted}") + if has_multiline and n < len(formatted_choices) - 1: + choices_lines.append("") # Blank line separator between options + + choices_list = "\n".join(choices_lines) + max_index = int(domain_obj.range_compatibility_identifier) - 1 # type: ignore[attr-defined] + print( + f"Please provide an index for '{current_path}'\n" + f"Choices:\n{choices_list}\n" + f"Valid range: [0, {max_index}]: " + ) + + while True: + sampled_value: str | int | float = input() + try: + if isinstance(domain_obj, Integer): + sampled_value = int(sampled_value) + elif isinstance(domain_obj, Float): + sampled_value = float(sampled_value) + elif isinstance(domain_obj, Categorical): + sampled_value = int(sampled_value) + else: + raise ValueError( + f"Unsupported domain type: {type(domain_obj).__name__}" + ) + + assert isinstance(domain_obj, Float | Integer | Categorical) + + if validate_parameter_value(domain_obj, sampled_value): + print(f"Value {sampled_value} recorded.\n") + break + else: + print( + f"Invalid value '{sampled_value}' for domain '{current_path}'. " + "Please try again: ", + ) + except ValueError: + print( + f"Could not convert input '{sampled_value}' to the required type. " + "Please try again: ", + ) + + return cast("T", sampled_value) + + def sample_environment_values(self, pipeline_space: P) -> Mapping[str, Any]: + """Get the environment values for the sampler. + + Returns: + The interactively chosen environment values. + """ + environment_values = {} + for fidelity_name, fidelity_object in pipeline_space.fidelity_attrs.items(): + domain_obj = fidelity_object.domain + print( + "Please provide" + f" {'a float' if isinstance(domain_obj, Float) else 'an integer'} value" + f" for the Fidelity '{fidelity_name}' in the range" + f" [{domain_obj.lower}, {domain_obj.upper}]: ", + ) + while True: + sampled_value: str | int | float = input() + try: + if isinstance(domain_obj, Integer): + sampled_value = int(sampled_value) + elif isinstance(domain_obj, Float): + sampled_value = float(sampled_value) + else: + raise ValueError( + f"Unsupported domain type: {type(domain_obj).__name__}" + ) + + if validate_parameter_value(domain_obj, sampled_value): + print(f"Value {sampled_value} recorded.\n") + break + else: + print( + f"Invalid value '{sampled_value}' for Fidelity" + f" '{fidelity_object!s}'. Please try again: ", + ) + except ValueError: + print( + f"Could not convert input '{sampled_value}' to the required type." + " Please try again: ", + ) + environment_values[fidelity_name] = sampled_value + + return environment_values + + +class PriorOrFallbackSampler(DomainSampler): + """A sampler that uses a prior value if available, otherwise falls back to another + sampler. + + Args: + fallback_sampler: A DomainSampler to use if the prior is not available. + always_use_prior: If True, always use the prior value when available. + """ + + def __init__( + self, + fallback_sampler: DomainSampler, + always_use_prior: bool = False, # noqa: FBT001, FBT002 + ): + """Initialize the sampler with a fallback sampler and a flag to always use the + prior. + + Args: + fallback_sampler: A DomainSampler to use if the prior is not available. + always_use_prior: If True, always use the prior value when available. + """ + self._fallback_sampler = fallback_sampler + self._always_use_prior = always_use_prior + + def __call__( + self, + *, + domain_obj: Domain[T], + current_path: str, + ) -> T: + """Sample a value from the domain, using the prior if available and according to + the prior confidence probability. + + Args: + domain_obj: The domain object from which to sample. + current_path: The path for which to sample a value. + + Returns: + A sampled value, either from the prior or from the fallback sampler. + + Raises: + ValueError: If the domain does not have a prior defined and the fallback + sampler is not provided. + """ + if domain_obj.has_prior: + _prior_probability = PRIOR_CONFIDENCE_MAPPING.get( + domain_obj.prior_confidence.value, 0.5 + ) + if isinstance(domain_obj, Categorical) or self._always_use_prior: + if ( + random.choices( + (True, False), + weights=(_prior_probability, 1 - _prior_probability), + k=1, + )[0] + or self._always_use_prior + ): + # If the prior is defined, we sample from it. + return domain_obj.prior + + # For Integers and Floats, sample gaussians around the prior + + elif isinstance(domain_obj, Integer | Float): + # Sample an integer from a Gaussian distribution centered around the + # prior, cut of the tails to ensure the value is within the domain's + # range. Using the _prior_probability to determine the standard deviation + assert hasattr(domain_obj, "lower") + assert hasattr(domain_obj, "upper") + assert hasattr(domain_obj, "prior") + + std_dev = 1 / ( + 10 * _prior_probability / (domain_obj.upper - domain_obj.lower) # type: ignore + ) + + a = (domain_obj.lower - domain_obj.prior) / std_dev # type: ignore + b = (domain_obj.upper - domain_obj.prior) / std_dev # type: ignore + sampled_value = stats.truncnorm.rvs( + a=a, + b=b, + loc=domain_obj.prior, # type: ignore + scale=std_dev, + ) + if isinstance(domain_obj, Integer): + sampled_value = round(sampled_value) + else: + sampled_value = float(sampled_value) # type: ignore + return cast("T", sampled_value) + + return self._fallback_sampler( + domain_obj=domain_obj, + current_path=current_path, + ) + + +def _mutate_samplings_to_make_by_forgetting( + samplings_to_make: Mapping[str, Any], + n_forgets: int, +) -> Mapping[str, Any]: + mutated_samplings_to_make = dict(**samplings_to_make) + + samplings_to_delete = random.sample( + list(samplings_to_make.keys()), + k=n_forgets, + ) + + for choice_to_delete in samplings_to_delete: + mutated_samplings_to_make.pop(choice_to_delete) + + return mutated_samplings_to_make + + +class MutateByForgettingSampler(DomainSampler): + """A sampler that mutates predefined samplings by forgetting a certain number of + them. It randomly selects a number of predefined samplings to forget and returns a + new sampler that only uses the remaining samplings. + + Args: + predefined_samplings: A mapping of paths to predefined values. + n_forgets: The number of predefined samplings to forget. + This should be an integer greater than 0 and less than or equal to the + number of predefined samplings. + + Raises: + ValueError: If n_forgets is not a valid integer or if it exceeds the number + of predefined samplings. + """ + + def __init__( + self, + predefined_samplings: Mapping[str, Any], + n_forgets: int, + ): + """Initialize the sampler with predefined samplings and a number of forgets. + + Args: + predefined_samplings: A mapping of paths to predefined values. + n_forgets: The number of predefined samplings to forget. + This should be an integer greater than 0 and less than or equal to the + number of predefined samplings. + + Raises: + ValueError: If n_forgets is not a valid integer or if it exceeds the + number of predefined samplings. + """ + if ( + not isinstance(n_forgets, int) + or n_forgets <= 0 + or n_forgets > len(predefined_samplings) + ): + raise ValueError(f"Invalid value for `n_forgets`: {n_forgets!r}.") + + mutated_samplings_to_make = _mutate_samplings_to_make_by_forgetting( + samplings_to_make=predefined_samplings, + n_forgets=n_forgets, + ) + + self._random_sampler = RandomSampler( + predefined_samplings=mutated_samplings_to_make, + ) + + def __call__( + self, + *, + domain_obj: Domain[T], + current_path: str, + ) -> T: + """Sample a value from the mutated predefined samplings or the domain. + + Args: + domain_obj: The domain object from which to sample. + current_path: The path for which to sample a value. + + Returns: + A sampled value, either from the mutated predefined samplings or from + the domain. + + Raises: + ValueError: If the current path is not in the mutated predefined + samplings and the domain does not have a prior defined. + """ + return self._random_sampler(domain_obj=domain_obj, current_path=current_path) + + +class MutatateUsingCentersSampler(DomainSampler): + """A sampler that mutates predefined samplings by forgetting a certain number of them, + but still uses the original values as centers for sampling. + + Args: + predefined_samplings: A mapping of paths to predefined values. + n_mutations: The number of predefined samplings to mutate. + This should be an integer greater than 0 and less than or equal to the number + of predefined samplings. + + Raises: + ValueError: If n_mutations is not a valid integer or if it exceeds the number + of predefined samplings. + """ + + def __init__( + self, + predefined_samplings: Mapping[str, Any], + n_mutations: int, + ): + """Initialize the sampler with predefined samplings and a number of mutations. + + Args: + predefined_samplings: A mapping of paths to predefined values. + n_mutations: The number of predefined samplings to mutate. + This should be an integer greater than 0 and less than or equal to the + number of predefined samplings. + + Raises: + ValueError: If n_mutations is not a valid integer or if it exceeds + the number of predefined samplings. + """ + if ( + not isinstance(n_mutations, int) + or n_mutations <= 0 + or n_mutations > len(predefined_samplings) + ): + raise ValueError(f"Invalid value for `n_mutations`: {n_mutations!r}.") + + self._kept_samplings_to_make = _mutate_samplings_to_make_by_forgetting( + samplings_to_make=predefined_samplings, + n_forgets=n_mutations, + ) + + # Still remember the original choices. We'll use them as centers later. + self._original_samplings_to_make = predefined_samplings + + def __call__( + self, + *, + domain_obj: Domain[T], + current_path: str, + ) -> T: + """Sample a value from the predefined samplings or the domain, using original + values as centers if the current path is not in the kept samplings. + + Args: + domain_obj: The domain object from which to sample. + current_path: The path for which to sample a value. + + Returns: + A sampled value, either from the kept samplings or from the domain, + using the original values as centers if necessary. + + Raises: + ValueError: If the current path is not in the kept samplings and the + domain does not have a prior defined. + """ + if current_path not in self._kept_samplings_to_make: + # For this path we either have forgotten the value or we never had it. + if current_path in self._original_samplings_to_make: + # We had a value for this path originally, use it as a center. + original_value = self._original_samplings_to_make[current_path] + sampled_value = domain_obj.centered_around( + center=original_value, + confidence=ConfidenceLevel.MEDIUM, + ).sample() + else: + # We never had a value for this path, we can only sample from the domain. + sampled_value = domain_obj.sample() + else: + # For this path we have chosen to keep the original value. + sampled_value = cast("T", self._kept_samplings_to_make[current_path]) + + return sampled_value + + +class CrossoverNotPossibleError(Exception): + """Exception raised when a crossover operation is not possible.""" + + +def _crossover_samplings_to_make_by_mixing( + predefined_samplings_1: Mapping[str, Any], + predefined_samplings_2: Mapping[str, Any], + prefer_first_probability: float, +) -> tuple[bool, Mapping[str, Any]]: + crossed_over_samplings = dict(**predefined_samplings_1) + made_any_crossovers = False + + for path, sampled_value_in_2 in predefined_samplings_2.items(): + if path in crossed_over_samplings: + use_value_from_2 = random.choices( + (False, True), + weights=(prefer_first_probability, 1 - prefer_first_probability), + k=1, + )[0] + if use_value_from_2: + crossed_over_samplings[path] = sampled_value_in_2 + made_any_crossovers = True + else: + crossed_over_samplings[path] = sampled_value_in_2 + + return made_any_crossovers, crossed_over_samplings + + +class CrossoverByMixingSampler(DomainSampler): + """A sampler that performs a crossover operation by mixing two sets of predefined + samplings. It combines the predefined samplings from two sources, allowing for a + probability-based selection of values from either source. + + Args: + predefined_samplings_1: The first set of predefined samplings. + predefined_samplings_2: The second set of predefined samplings. + prefer_first_probability: The probability of preferring values from the first + set over the second set when both have values for the same path. + This should be a float between 0 and 1, where 0 means always prefer the + second set and 1 means always prefer the first set. + + Raises: + ValueError: If prefer_first_probability is not between 0 and 1. + CrossoverNotPossibleError: If no crossovers were made between the two sets + of predefined samplings. + """ + + def __init__( + self, + predefined_samplings_1: Mapping[str, Any], + predefined_samplings_2: Mapping[str, Any], + prefer_first_probability: float, + ): + """Initialize the sampler with two sets of predefined samplings and a preference + probability for the first set. + + Args: + predefined_samplings_1: The first set of predefined samplings. + predefined_samplings_2: The second set of predefined samplings. + prefer_first_probability: The probability of preferring values from the + first set over the second set when both have values for the same path. + This should be a float between 0 and 1, where 0 means always prefer the + second set and 1 means always prefer the first set. + + Raises: + ValueError: If prefer_first_probability is not between 0 and 1. + """ + if not isinstance(prefer_first_probability, float) or not ( + 0 <= prefer_first_probability <= 1 + ): + raise ValueError( + "Invalid value for `prefer_first_probability`:" + f" {prefer_first_probability!r}." + ) + + ( + made_any_crossovers, + crossed_over_samplings_to_make, + ) = _crossover_samplings_to_make_by_mixing( + predefined_samplings_1=predefined_samplings_1, + predefined_samplings_2=predefined_samplings_2, + prefer_first_probability=prefer_first_probability, + ) + + if not made_any_crossovers: + raise CrossoverNotPossibleError("No crossovers were made.") + + self._random_sampler = RandomSampler( + predefined_samplings=crossed_over_samplings_to_make, + ) + + def __call__( + self, + *, + domain_obj: Domain[T], + current_path: str, + ) -> T: + """Sample a value from the crossed-over predefined samplings or the domain. + + Args: + domain_obj: The domain object from which to sample. + current_path: The path for which to sample a value. + + Returns: + A sampled value, either from the crossed-over predefined samplings or + from the domain. + + Raises: + ValueError: If the current path is not in the crossed-over predefined + samplings and the domain does not have a prior defined. + """ + return self._random_sampler(domain_obj=domain_obj, current_path=current_path) diff --git a/neps/space/neps_spaces/string_formatter.py b/neps/space/neps_spaces/string_formatter.py new file mode 100644 index 000000000..0f6da81b2 --- /dev/null +++ b/neps/space/neps_spaces/string_formatter.py @@ -0,0 +1,511 @@ +"""Pretty formatting for Operation objects. + +This module provides functionality to convert Operation objects into +human-readable formatted strings. The format is Pythonic and preserves +all information including nested operations, lists, tuples, and dicts. + +ARCHITECTURE: + format_value() - Single entry point for ALL formatting + ├── _format_categorical() - Internal handler for Categorical + ├── _format_float() - Internal handler for Float + ├── _format_integer() - Internal handler for Integer + ├── _format_resampled() - Internal handler for Resample + ├── _format_repeated() - Internal handler for Repeated + ├── _format_operation() - Internal handler for Operation + ├── _format_sequence() - Internal handler for list/tuple + └── _format_dict() - Internal handler for dict + +All __str__ methods should call format_value() directly. +All internal formatters call format_value() for nested values. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from neps.space.neps_spaces.parameters import ( + ByName, + Categorical, + Float, + Integer, + Operation, + Repeated, + Resample, + ) + + +@dataclass +class FormatterStyle: + """Configuration for the formatting style.""" + + indent_str: str = " " # Three spaces for indentation + max_line_length: int = 90 # Try to keep lines under this length + compact_threshold: int = 40 # Use compact format if repr is shorter + show_empty_args: bool = True # Show () for operations with no args/kwargs + + +# ============================================================================ +# PUBLIC API - Single entry point for all formatting +# ============================================================================ + + +def format_value( # noqa: C901, PLR0911, PLR0912 + value: Any, + indent: int = 0, + style: FormatterStyle | None = None, +) -> str: + """Format any value with proper indentation and style. + + This is the SINGLE entry point for all formatting in NePS. + All __str__ methods should delegate to this function. + + Args: + value: The value to format (any type) + indent: Current indentation level + style: Formatting style configuration + + Returns: + Formatted string representation + """ + from neps.space.neps_spaces.parameters import ( + ByName, + Categorical, + Fidelity, + Float, + Integer, + Operation, + Repeated, + Resample, + ) + + if style is None: + style = FormatterStyle() + + # Dispatch to appropriate internal formatter based on type + if isinstance(value, Operation): + return _format_operation(value, indent, style) + + if isinstance(value, Categorical): + return _format_categorical(value, indent, style) + + if isinstance(value, Float): + return _format_float(value, indent, style) + + if isinstance(value, Integer): + return _format_integer(value, indent, style) + + if isinstance(value, Fidelity): + # Use the __str__ method of Fidelity subclasses directly + return str(value) + + if isinstance(value, ByName): + return _format_by_name(value, indent, style) + + if isinstance(value, Resample): + return _format_resampled(value, indent, style) + + if isinstance(value, Repeated): + return _format_repeated(value, indent, style) + + if isinstance(value, list | tuple): + return _format_sequence(value, indent, style) + + if isinstance(value, dict): + return _format_dict(value, indent, style) + + # Check for PipelineSpace (import here to avoid circular dependency) + from neps.space.neps_spaces.parameters import PipelineSpace + + if isinstance(value, PipelineSpace): + return _format_pipeline_space(value, indent, style) + + # For callables (functions, methods), show their name + if callable(value) and (name := getattr(value, "__name__", None)): + return name + + # For identifier strings, don't add quotes + if isinstance(value, str) and value.isidentifier(): + return value + + # For other values, use repr + return repr(value) + + +# ============================================================================ +# HELPER FUNCTIONS +# ============================================================================ + + +def _collapse_closing_brackets(text: str) -> str: + """Collapse consecutive closing brackets onto same line respecting indentation. + + Transforms: + ) + ) + ) + Into: + ) ) ) + + All brackets are placed on the same line using the minimum indentation. + + Args: + text: The formatted text + + Returns: + Text with collapsed closing brackets + """ + lines = text.split("\n") + result = [] + i = 0 + + while i < len(lines): + current_line = lines[i] + stripped = current_line.strip() + + # Check if this line contains only closing brackets + if stripped and all(c in ")]" for c in stripped): + # Collect consecutive bracket lines + bracket_lines = [current_line] + j = i + 1 + while ( + j < len(lines) + and lines[j].strip() + and all(c in ")]" for c in lines[j].strip()) + ): + bracket_lines.append(lines[j]) + j += 1 + + # Collapse if multiple bracket lines + if len(bracket_lines) > 1: + # Find minimum indentation + min_indent = min(len(line) - len(line.lstrip()) for line in bracket_lines) + # Collapse onto single line + combined = " ".join(line.strip() for line in bracket_lines) + result.append(" " * min_indent + combined) + else: + result.append(current_line) + + i = j + else: + result.append(current_line) + i += 1 + + return "\n".join(result) + + +# ============================================================================ +# INTERNAL FORMATTERS - Type-specific formatting logic +# All these call format_value() for nested values to maintain consistency +# ============================================================================ + + +def _format_prior_confidence(prior_confidence: Any) -> str: + """Internal helper to format prior_confidence values consistently. + + Args: + prior_confidence: The prior confidence value (typically a ConfidenceLevel enum) + + Returns: + String representation of the prior confidence + """ + return ( + prior_confidence.value + if hasattr(prior_confidence, "value") + else str(prior_confidence) + ) + + +def _format_categorical( + categorical: Categorical, + indent: int, + style: FormatterStyle, +) -> str: + """Internal formatter for Categorical parameters.""" + indent_str = style.indent_str * indent + inner_indent_str = style.indent_str * (indent + 1) + choice_indent_str = style.indent_str * (indent + 2) + + # Format each choice using format_value for consistency + formatted_choices = [] + for choice in categorical.choices: # type: ignore[union-attr] + choice_str = format_value(choice, indent + 2, style) + formatted_choices.append(choice_str) + + # Check if all choices are simple (strings or numbers without newlines) + all_simple = all("\n" not in choice_str for choice_str in formatted_choices) + + if all_simple and formatted_choices: + # Try to fit choices on one line + choices_str = ", ".join(formatted_choices) + if len(choices_str) <= style.max_line_length: + # Put choices on own line, indented + result = f"Categorical(\n{inner_indent_str}choices=({choices_str})" + else: + # Put on multiple lines but keep choices together + choices_str = f",\n{choice_indent_str}".join(formatted_choices) + result = ( + f"Categorical(\n{inner_indent_str}choices=(\n" + f"{choice_indent_str}{choices_str})" + ) + else: + # Complex choices - use multi-line format + choices_str = f",\n{choice_indent_str}".join(formatted_choices) + result = ( + f"Categorical(\n{inner_indent_str}choices=(\n" + f"{choice_indent_str}{choices_str}\n{inner_indent_str})" + ) + + if categorical.has_prior: + prior_confidence_str = _format_prior_confidence(categorical._prior_confidence) + result += ( + f",\n{inner_indent_str}prior={categorical._prior}," + f"\n{inner_indent_str}prior_confidence={prior_confidence_str}" + ) + + result += f"\n{indent_str})" + return _collapse_closing_brackets(result) + + +def _format_float( + float_param: Float, + indent: int, # noqa: ARG001 + style: FormatterStyle, # noqa: ARG001 +) -> str: + """Internal formatter for Float parameters.""" + string = f"Float({float_param._lower}, {float_param._upper}" + if float_param._log: + string += ", log" + if float_param.has_prior: + prior_confidence_str = _format_prior_confidence(float_param._prior_confidence) + string += f", prior={float_param._prior}, prior_confidence={prior_confidence_str}" + string += ")" + return string + + +def _format_integer( + integer_param: Integer, + indent: int, # noqa: ARG001 + style: FormatterStyle, # noqa: ARG001 +) -> str: + """Internal formatter for Integer parameters.""" + string = f"Integer({integer_param._lower}, {integer_param._upper}" + if integer_param._log: + string += ", log" + if integer_param.has_prior: + prior_confidence_str = _format_prior_confidence(integer_param._prior_confidence) + string += ( + f", prior={integer_param._prior}, prior_confidence={prior_confidence_str}" + ) + string += ")" + return string + + +def _format_by_name( + by_name: ByName, + indent: int, # noqa: ARG001 + style: FormatterStyle, # noqa: ARG001 +) -> str: + """Internal formatter for ByName references.""" + return f'ByName("{by_name.name}")' + + +def _format_resampled( + resampled: Resample, + indent: int, + style: FormatterStyle, +) -> str: + """Internal formatter for Resample parameters.""" + source = resampled._source + + # Format the source using unified format_value + source_str = format_value(source, indent + 1, style) + + # Use multi-line format if source is multi-line + if "\n" in source_str: + indent_str = style.indent_str * indent + inner_indent_str = style.indent_str * (indent + 1) + result = f"Resample(\n{inner_indent_str}{source_str}\n{indent_str})" + return _collapse_closing_brackets(result) + + # Simple single-line format for basic types + return f"Resample({source_str})" + + +def _format_repeated( + repeated: Repeated, + indent: int, + style: FormatterStyle, +) -> str: + """Internal formatter for Repeated parameters.""" + source_str = format_value(repeated._content, indent, style) + return f"Repeated({source_str})" + + +def _format_sequence( + seq: list | tuple, + indent: int, + style: FormatterStyle, +) -> str: + """Internal formatter for lists and tuples.""" + from neps.space.neps_spaces.parameters import Operation + + if not seq: + return "[]" if isinstance(seq, list) else "()" + + # Format all items + formatted_items = [format_value(item, indent + 1, style) for item in seq] + + # Check for "Nx" shorthand case (all items identical) + if len(set(formatted_items)) == 1 and len(seq) > 1: + return f"{len(seq)}x {formatted_items[0]}" + + # Try compact format for simple sequences + compact = repr(seq) + if len(compact) <= style.compact_threshold and "\n" not in compact: + return compact + + # Expand multi-line or complex sequences + is_list = isinstance(seq, list) + bracket_open, bracket_close = ("[", "]") if is_list else ("(", ")") + indent_str = style.indent_str * indent + inner_indent_str = style.indent_str * (indent + 1) + + # Check if expansion is needed (Operations or multi-line items) + needs_expansion = any( + isinstance(item, Operation) or "\n" in item_str + for item, item_str in zip(seq, formatted_items, strict=False) + ) + + if needs_expansion: + # Full expansion: each item on its own line + lines = [bracket_open] + lines.extend(f"{inner_indent_str}{item}," for item in formatted_items) + lines.append(f"{indent_str}{bracket_close}") + else: + # Compact expansion: fit multiple items per line + lines = [bracket_open] + current_line: list[str] = [] + current_length = 0 + + for item_str in formatted_items: + item_len = len(item_str) + 2 # +2 for ", " + if current_line and current_length + item_len > style.max_line_length: + lines.append(f"{inner_indent_str}{', '.join(current_line)},") + current_line, current_length = [item_str], len(item_str) + else: + current_line.append(item_str) + current_length += item_len + + if current_line: + lines.append(f"{inner_indent_str}{', '.join(current_line)},") + lines.append(f"{indent_str}{bracket_close}") + + result = "\n".join(lines) + return _collapse_closing_brackets(result) + + +def _format_dict( + d: dict, + indent: int, + style: FormatterStyle, +) -> str: + """Internal formatter for dictionaries.""" + if not d: + return "{}" + + # Try compact format first + compact = repr(d) + if len(compact) <= style.compact_threshold: + return compact + + # Use expanded format + indent_str = style.indent_str * indent + inner_indent_str = style.indent_str * (indent + 1) + + lines = ["{"] + for key, value in d.items(): + formatted_value = format_value(value, indent + 1, style) + lines.append(f"{inner_indent_str}{key!r}: {formatted_value},") + lines.append(f"{indent_str}}}") + return "\n".join(lines) + + +def _format_operation( + operation: Operation, + indent: int, + style: FormatterStyle, +) -> str: + """Internal formatter for Operation objects.""" + # Get operator name + operator_name = ( + operation.operator + if isinstance(operation.operator, str) + else operation.operator.__name__ + ) + + # Helper to safely get args/kwargs, handling unresolved Resolvables + def safe_get_args() -> tuple[Any, ...]: + """Get args, handling unresolved Resolvables by wrapping if needed.""" + try: + return operation.args + except ValueError: + # Args contain unresolved Resolvables, use raw _args + args = operation._args + # Wrap single Resolvable in tuple for iteration + return (args,) if not isinstance(args, tuple | list) else args + + def safe_get_kwargs() -> dict[str, Any]: + """Get kwargs, handling unresolved Resolvables.""" + try: + return dict(operation.kwargs) + except ValueError: + # Kwargs contain unresolved Resolvables, skip or use raw _kwargs + kwargs = operation._kwargs + return kwargs if isinstance(kwargs, dict) else {} + + # Check if we have any content + has_args = bool(safe_get_args()) + has_kwargs = bool(safe_get_kwargs()) + + if not (has_args or has_kwargs): + return f"{operator_name}()" if style.show_empty_args else operator_name + + # Format with multi-line layout + indent_str = style.indent_str * indent + inner_indent_str = style.indent_str * (indent + 1) + + lines = [f"{operator_name}("] + + for arg in safe_get_args(): + formatted = format_value(arg, indent + 1, style) + lines.append(f"{inner_indent_str}{formatted},") + + for key, value in safe_get_kwargs().items(): + formatted_value = format_value(value, indent + 1, style) + lines.append(f"{inner_indent_str}{key}={formatted_value},") + + lines.append(f"{indent_str})") + + return "\n".join(lines) + + +def _format_pipeline_space( + pipeline_space: Any, + indent: int, # noqa: ARG001 + style: FormatterStyle, +) -> str: + """Internal formatter for PipelineSpace objects.""" + lines = [f"{pipeline_space.__class__.__name__} with parameters:"] + for k, v in pipeline_space.get_attrs().items(): + if not k.startswith("_") and not callable(v): + # Use the unified formatter for all values + formatted_value = format_value(v, 0, style) + # If multi-line, indent all lines + if "\n" in formatted_value: + indented_value = "\n ".join(formatted_value.split("\n")) + lines.append(f" {k}:\n {indented_value}") + else: + lines.append(f" {k} = {formatted_value}") + return "\n".join(lines) diff --git a/neps/space/parameters.py b/neps/space/parameters.py index 868c10c0e..723b1fd38 100644 --- a/neps/space/parameters.py +++ b/neps/space/parameters.py @@ -12,7 +12,7 @@ @dataclass -class Float: +class HPOFloat: """A float value for a parameter. This kind of parameter is used to represent hyperparameters with continuous float @@ -56,19 +56,19 @@ class Float: def __post_init__(self) -> None: if self.lower >= self.upper: raise ValueError( - f"Float parameter: bounds error (lower >= upper). Actual values: " + "Float parameter: bounds error (lower >= upper). Actual values: " f"lower={self.lower}, upper={self.upper}" ) if self.log and (self.lower <= 0 or self.upper <= 0): raise ValueError( - f"Float parameter: bounds error (log scale cant have bounds <= 0). " + "Float parameter: bounds error (log scale cant have bounds <= 0). " f"Actual values: lower={self.lower}, upper={self.upper}" ) if self.prior is not None and not self.lower <= self.prior <= self.upper: raise ValueError( - f"Float parameter: prior bounds error. Expected lower <= prior <= upper, " + "Float parameter: prior bounds error. Expected lower <= prior <= upper, " f"but got lower={self.lower}, prior={self.prior}, upper={self.upper}" ) @@ -77,14 +77,14 @@ def __post_init__(self) -> None: if self.is_fidelity and (self.lower < 0 or self.upper < 0): raise ValueError( - f"Float parameter: fidelity bounds error. Expected fidelity" + "Float parameter: fidelity bounds error. Expected fidelity" f" bounds to be >= 0, but got lower={self.lower}, " f" upper={self.upper}." ) if self.is_fidelity and self.prior is not None: raise ValueError( - f"Float parameter: Fidelity parameters " + "Float parameter: Fidelity parameters " f"cannot have a prior value. Got prior={self.prior}." ) @@ -97,13 +97,9 @@ def __post_init__(self) -> None: self.domain = Domain.floating(self.lower, self.upper, log=self.log) self.center = self.domain.cast_one(0.5, frm=Domain.unit_float()) - def validate(self, value: Any) -> bool: - """Validate if a value is within the bounds of the float parameter.""" - return isinstance(value, float | int) and self.lower <= value <= self.upper - @dataclass -class Integer: +class HPOInteger: """An integer value for a parameter. This kind of parameter is used to represent hyperparameters with @@ -147,7 +143,7 @@ class Integer: def __post_init__(self) -> None: if self.lower >= self.upper: raise ValueError( - f"Integer parameter: bounds error (lower >= upper). Actual values: " + "Integer parameter: bounds error (lower >= upper). Actual values: " f"lower={self.lower}, upper={self.upper}" ) @@ -159,7 +155,7 @@ def __post_init__(self) -> None: upper_int = int(self.upper) if lower_int != self.lower or upper_int != self.upper: raise ValueError( - f"Integer parameter: bounds error (lower and upper must be integers). " + "Integer parameter: bounds error (lower and upper must be integers). " f"Actual values: lower={self.lower}, upper={self.upper}" ) @@ -168,39 +164,35 @@ def __post_init__(self) -> None: if self.is_fidelity and (self.lower < 0 or self.upper < 0): raise ValueError( - f"Integer parameter: fidelity bounds error. Expected fidelity" + "Integer parameter: fidelity bounds error. Expected fidelity" f" bounds to be >= 0, but got lower={self.lower}, " f" upper={self.upper}." ) if self.log and (self.lower <= 0 or self.upper <= 0): raise ValueError( - f"Integer parameter: bounds error (log scale cant have bounds <= 0). " + "Integer parameter: bounds error (log scale cant have bounds <= 0). " f"Actual values: lower={self.lower}, upper={self.upper}" ) if self.prior is not None and not self.lower <= self.prior <= self.upper: raise ValueError( - f"Integer parameter: Expected lower <= prior <= upper," + "Integer parameter: Expected lower <= prior <= upper," f"but got lower={self.lower}, prior={self.prior}, upper={self.upper}" ) if self.is_fidelity and self.prior is not None: raise ValueError( - f"Integer parameter: Fidelity parameters " + "Integer parameter: Fidelity parameters " f"cannot have a prior value. Got prior={self.prior}." ) self.domain = Domain.integer(self.lower, self.upper, log=self.log) self.center = self.domain.cast_one(0.5, frm=Domain.unit_float()) - def validate(self, value: Any) -> bool: - """Validate if a value is within the bounds of the parameter.""" - return isinstance(value, float | int) and self.lower <= value <= self.upper - @dataclass -class Categorical: +class HPOCategorical: """A list of **unordered** choices for a parameter. This kind of parameter is used to represent hyperparameters that can take on a @@ -260,13 +252,9 @@ def __post_init__(self) -> None: self.center = self.choices[0] self.domain = Domain.indices(len(self.choices), is_categorical=True) - def validate(self, value: Any) -> bool: - """Validate if a value is one of the choices of the categorical parameter.""" - return value in self.choices - @dataclass -class Constant: +class HPOConstant: """A constant value for a parameter. This kind of parameter is used to represent hyperparameters with values that @@ -295,17 +283,13 @@ def center(self) -> Any: """ return self.value - def validate(self, value: Any) -> bool: - """Validate if a value is the same as the constant parameter's value.""" - return value == self.value - -Parameter: TypeAlias = Float | Integer | Categorical +Parameter: TypeAlias = HPOFloat | HPOInteger | HPOCategorical """A type alias for all the parameter types. -* [`Float`][neps.space.Float] -* [`Integer`][neps.space.Integer] -* [`Categorical`][neps.space.Categorical] +* [`Float`][neps.space.HPOFloat] +* [`Integer`][neps.space.HPOInteger] +* [`Categorical`][neps.space.HPOCategorical] -A [`Constant`][neps.space.Constant] is not included as it does not change value. +A [`Constant`][neps.space.HPOConstant] is not included as it does not change value. """ diff --git a/neps/space/parsing.py b/neps/space/parsing.py index 6c46e7b4a..ef088ca21 100644 --- a/neps/space/parsing.py +++ b/neps/space/parsing.py @@ -9,7 +9,19 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, TypeAlias -from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter +from neps.space.neps_spaces.parameters import ( + Categorical, + Float, + Integer, + PipelineSpace, +) +from neps.space.parameters import ( + HPOCategorical, + HPOConstant, + HPOFloat, + HPOInteger, + Parameter, +) from neps.space.search_space import SearchSpace if TYPE_CHECKING: @@ -55,7 +67,9 @@ def scientific_parse(value: str | int | float) -> str | int | float: ) -def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: C901, PLR0911, PLR0912 +def as_parameter( # noqa: C901, PLR0911, PLR0912 + details: SerializedParameter, +) -> Parameter | HPOConstant: """Deduces the parameter type from details. Args: @@ -73,7 +87,7 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: # Constant case str() | int() | float(): val = scientific_parse(details) - return Constant(val) + return HPOConstant(val) # Bounds of float or int case tuple((x, y)): @@ -81,9 +95,9 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: _y = scientific_parse(y) match (_x, _y): case (int(), int()): - return Integer(_x, _y) + return HPOInteger(_x, _y) case (float(), float()): - return Float(_x, _y) + return HPOFloat(_x, _y) case _: raise ValueError( f"Expected both 'int' or 'float' for bounds but got {type(_x)=}" @@ -100,9 +114,9 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: _y = scientific_parse(y) match (_x, _y): case (int(), int()) if _x <= _y: # 2./3. - return Integer(_x, _y) + return HPOInteger(_x, _y) case (float(), float()) if _x <= _y: # 2./3. - return Float(_x, _y) + return HPOFloat(_x, _y) # Error case: # We do have two numbers, but of different types. This could @@ -120,7 +134,7 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: ) # At least one of them is a string, so we treat is as categorical. case _: - return Categorical(choices=[_x, _y]) + return HPOCategorical(choices=[_x, _y]) ## Categorical list of choices (tuple is reserved for bounds) case Sequence() if not isinstance(details, tuple): @@ -129,7 +143,7 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: # when specifying a grid. Hence, we map over the list and convert # what we can details = [scientific_parse(d) for d in details] - return Categorical(details) + return HPOCategorical(details) # Categorical dict declartion case {"choices": choices, **rest}: @@ -139,7 +153,7 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: # See note above about scientific notation elements choices = [scientific_parse(c) for c in choices] - return Categorical(choices, **rest) # type: ignore + return HPOCategorical(choices, **rest) # type: ignore # Constant dict declartion case {"value": v, **_rest}: @@ -150,7 +164,7 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: f" which indicates to treat value `{v}` a constant." ) - return Constant(v, **_rest) # type: ignore + return HPOConstant(v, **_rest) # type: ignore # Bounds dict declartion case {"lower": l, "upper": u, **rest}: @@ -160,18 +174,18 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: _type = rest.pop("type", None) match _type: case "int" | "integer": - return Integer(_x, _y, **rest) # type: ignore + return HPOInteger(_x, _y, **rest) # type: ignore case "float" | "floating": - return Float(_x, _y, **rest) # type: ignore + return HPOFloat(_x, _y, **rest) # type: ignore case None: match (_x, _y): case (int(), int()): - return Integer(_x, _y, **rest) # type: ignore + return HPOInteger(_x, _y, **rest) # type: ignore case (float(), float()): - return Float(_x, _y, **rest) # type: ignore + return HPOFloat(_x, _y, **rest) # type: ignore case _: raise ValueError( - f"Expected both 'int' or 'float' for bounds but" + "Expected both 'int' or 'float' for bounds but" f" got {type(_x)=} and {type(_y)=}." ) case _: @@ -188,18 +202,21 @@ def as_parameter(details: SerializedParameter) -> Parameter | Constant: # noqa: def convert_mapping(pipeline_space: Mapping[str, Any]) -> SearchSpace: """Converts a dictionary to a SearchSpace object.""" - parameters: dict[str, Parameter | Constant] = {} + parameters: dict[str, Parameter | HPOConstant] = {} for name, details in pipeline_space.items(): match details: - case Float() | Integer() | Categorical() | Constant(): + case HPOFloat() | HPOInteger() | HPOCategorical() | HPOConstant(): parameters[name] = dataclasses.replace(details) # copy + # New PipelineSpace parameters - converted by SearchSpace.__post_init__ + case Float() | Integer() | Categorical(): + parameters[name] = details # type: ignore[assignment] case str() | int() | float() | Mapping(): try: parameters[name] = as_parameter(details) except (TypeError, ValueError) as e: raise ValueError(f"Error parsing parameter '{name}'") from e case None: - parameters[name] = Constant(None) + parameters[name] = HPOConstant(None) case _: raise ValueError( f"Unrecognized parameter type '{type(details)}' for '{name}'." @@ -220,8 +237,12 @@ def convert_configspace(configspace: ConfigurationSpace) -> SearchSpace: """ import ConfigSpace as CS - space: dict[str, Parameter | Constant] = {} - if any(configspace.conditions) or any(configspace.forbidden_clauses): + space: dict[str, Parameter | HPOConstant] = {} + if ( + hasattr(configspace, "conditions") + and hasattr(configspace, "forbidden_clauses") + and (any(configspace.conditions) or any(configspace.forbidden_clauses)) + ): raise NotImplementedError( "The ConfigurationSpace has conditions or forbidden clauses, " "which are not supported by neps." @@ -230,9 +251,9 @@ def convert_configspace(configspace: ConfigurationSpace) -> SearchSpace: for name, hyperparameter in configspace.items(): match hyperparameter: case CS.Constant(): - space[name] = Constant(value=hyperparameter.value) + space[name] = HPOConstant(value=hyperparameter.value) case CS.CategoricalHyperparameter(): - space[name] = Categorical(hyperparameter.choices) # type: ignore + space[name] = HPOCategorical(hyperparameter.choices) # type: ignore case CS.OrdinalHyperparameter(): raise ValueError( "NePS does not support ordinals yet, please" @@ -240,14 +261,14 @@ def convert_configspace(configspace: ConfigurationSpace) -> SearchSpace: " categorical hyperparameter." ) case CS.UniformIntegerHyperparameter(): - space[name] = Integer( + space[name] = HPOInteger( lower=hyperparameter.lower, upper=hyperparameter.upper, log=hyperparameter.log, prior=None, ) case CS.UniformFloatHyperparameter(): - space[name] = Float( + space[name] = HPOFloat( lower=hyperparameter.lower, upper=hyperparameter.upper, log=hyperparameter.log, @@ -263,7 +284,7 @@ def convert_configspace(configspace: ConfigurationSpace) -> SearchSpace: UserWarning, stacklevel=2, ) - space[name] = Float( + space[name] = HPOFloat( lower=hyperparameter.lower, upper=hyperparameter.upper, log=hyperparameter.log, @@ -278,7 +299,7 @@ def convert_configspace(configspace: ConfigurationSpace) -> SearchSpace: UserWarning, stacklevel=2, ) - space[name] = Integer( + space[name] = HPOInteger( lower=hyperparameter.lower, upper=hyperparameter.upper, log=hyperparameter.log, @@ -295,8 +316,9 @@ def convert_to_space( Mapping[str, dict | str | int | float | Parameter] | SearchSpace | ConfigurationSpace + | PipelineSpace ), -) -> SearchSpace: +) -> SearchSpace | PipelineSpace: """Converts a search space to a SearchSpace object. Args: @@ -305,7 +327,7 @@ def convert_to_space( Returns: The SearchSpace object representing the search space. """ - # We quickly check ConfigSpace becuse it inherits from Mapping + # We quickly check ConfigSpace because it inherits from Mapping try: from ConfigSpace import ConfigurationSpace @@ -319,6 +341,8 @@ def convert_to_space( return space case Mapping(): return convert_mapping(space) + case PipelineSpace(): + return space case _: raise ValueError( f"Unsupported type '{type(space)}' for conversion to SearchSpace." diff --git a/neps/space/search_space.py b/neps/space/search_space.py index 2b0659f6a..3f4a4dcf0 100644 --- a/neps/space/search_space.py +++ b/neps/space/search_space.py @@ -3,29 +3,45 @@ any fidelities and constants. """ +# mypy: disable-error-code="unreachable" + from __future__ import annotations from collections.abc import Iterator, Mapping from dataclasses import dataclass, field from typing import Any -from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter +from neps.space.neps_spaces.parameters import ( + Categorical as PSCategorical, + ConfidenceLevel, + Float as PSFloat, + Integer as PSInteger, +) +from neps.space.parameters import ( + HPOCategorical, + HPOConstant, + HPOFloat, + HPOInteger, + Parameter, +) # NOTE: The use of `Mapping` instead of `dict` is so that type-checkers # can check if we accidetally mutate these as we pass the parameters around. # We really should not, and instead make a copy if we really need to. @dataclass -class SearchSpace(Mapping[str, Parameter | Constant]): +class SearchSpace( + Mapping[str, Parameter | HPOConstant | PSCategorical | PSFloat | PSInteger] +): """A container for parameters.""" - elements: Mapping[str, Parameter | Constant] = field(default_factory=dict) + elements: Mapping[str, Parameter | HPOConstant] = field(default_factory=dict) """All items in the search space.""" - categoricals: Mapping[str, Categorical] = field(init=False) + categoricals: Mapping[str, HPOCategorical] = field(init=False) """The categorical hyperparameters in the search space.""" - numerical: Mapping[str, Integer | Float] = field(init=False) + numerical: Mapping[str, HPOInteger | HPOFloat] = field(init=False) """The numerical hyperparameters in the search space. !!! note @@ -33,7 +49,7 @@ class SearchSpace(Mapping[str, Parameter | Constant]): This does not include fidelities. """ - fidelities: Mapping[str, Integer | Float] = field(init=False) + fidelities: Mapping[str, HPOInteger | HPOFloat] = field(init=False) """The fidelities in the search space. Currently no optimizer supports multiple fidelities but it is defined here incase. @@ -53,23 +69,106 @@ def searchables(self) -> Mapping[str, Parameter]: return {**self.numerical, **self.categoricals} @property - def fidelity(self) -> tuple[str, Float | Integer] | None: + def fidelity(self) -> tuple[str, HPOFloat | HPOInteger] | None: """The fidelity parameter for the search space.""" return None if len(self.fidelities) == 0 else next(iter(self.fidelities.items())) - def __post_init__(self) -> None: + def __post_init__(self) -> None: # noqa: C901, PLR0912, PLR0915 + # Convert new PipelineSpace parameters to HPO equivalents if needed + converted_elements = {} + for name, hp in self.elements.items(): + # Check if it's a new PipelineSpace parameter + if isinstance(hp, PSFloat | PSInteger | PSCategorical): + # The user will get a warning from neps.run about using SearchSpace + if isinstance(hp, PSFloat): + # Extract prior if it exists + prior: float | None = None + if hp.has_prior and hp.prior is not None: + prior = float(hp.prior) + # Get string value from ConfidenceLevel enum + prior_confidence: str = "low" + if hp.has_prior: + conf = hp.prior_confidence + prior_confidence = ( + conf.value if isinstance(conf, ConfidenceLevel) else conf + ) + converted_elements[name] = HPOFloat( + lower=float(hp.lower), + upper=float(hp.upper), + log=hp.log, + prior=prior, + prior_confidence=prior_confidence, # type: ignore[arg-type] + is_fidelity=getattr(hp, "_is_fidelity_compat", False), + ) + elif isinstance(hp, PSInteger): + # Extract prior if it exists + prior_int: int | None = None + if hp.has_prior and hp.prior is not None: + prior_int = int(hp.prior) + # Get string value from ConfidenceLevel enum + prior_confidence_int: str = "low" + if hp.has_prior: + conf = hp.prior_confidence + prior_confidence_int = ( + conf.value if isinstance(conf, ConfidenceLevel) else conf + ) + converted_elements[name] = HPOInteger( + lower=int(hp.lower), + upper=int(hp.upper), + log=hp.log, + prior=prior_int, + prior_confidence=prior_confidence_int, # type: ignore[arg-type] + is_fidelity=getattr(hp, "_is_fidelity_compat", False), + ) + elif isinstance(hp, PSCategorical): + # Categorical conversion - extract choices as list + # For SearchSpace, choices should be simple list[float | int | str] + if isinstance(hp.choices, tuple): + choices: list[float | int | str] = list(hp.choices) # type: ignore[arg-type] + else: + # If it's a Domain or complex structure, we can't easily convert + # Just try to use it as-is and let HPOCategorical validate + choices = list(hp.choices) # type: ignore[arg-type, assignment] + + # Extract prior if it exists + # In PipelineSpace, prior is index; in SearchSpace, actual value + prior_cat: float | int | str | None = None + if ( + hp.has_prior + and isinstance(hp.prior, int) + and 0 <= hp.prior < len(choices) + ): + # Convert index to actual choice + prior_cat = choices[hp.prior] # type: ignore[assignment] + + # Get string value from ConfidenceLevel enum + prior_confidence_cat: str = "low" + if hp.has_prior: + conf = hp.prior_confidence + prior_confidence_cat = ( + conf.value if isinstance(conf, ConfidenceLevel) else conf + ) + + converted_elements[name] = HPOCategorical( + choices=choices, + prior=prior_cat, + prior_confidence=prior_confidence_cat, # type: ignore[arg-type] + ) + else: + converted_elements[name] = hp + # Ensure that we have a consistent order for all our items. - self.elements = dict(sorted(self.elements.items(), key=lambda x: x[0])) + self.elements = dict(sorted(converted_elements.items(), key=lambda x: x[0])) - fidelities: dict[str, Float | Integer] = {} - numerical: dict[str, Float | Integer] = {} - categoricals: dict[str, Categorical] = {} + fidelities: dict[str, HPOFloat | HPOInteger] = {} + numerical: dict[str, HPOFloat | HPOInteger] = {} + categoricals: dict[str, HPOCategorical] = {} constants: dict[str, Any] = {} # Process the hyperparameters for name, hp in self.elements.items(): match hp: - case Float() | Integer() if hp.is_fidelity: + case HPOFloat() | HPOInteger() if hp.is_fidelity: # We should allow this at some point, but until we do, # raise an error if len(fidelities) >= 1: @@ -80,11 +179,11 @@ def __post_init__(self) -> None: ) fidelities[name] = hp - case Float() | Integer(): + case HPOFloat() | HPOInteger(): numerical[name] = hp - case Categorical(): + case HPOCategorical(): categoricals[name] = hp - case Constant(): + case HPOConstant(): constants[name] = hp.value case _: @@ -95,7 +194,7 @@ def __post_init__(self) -> None: self.constants = constants self.fidelities = fidelities - def __getitem__(self, key: str) -> Parameter | Constant: + def __getitem__(self, key: str) -> Parameter | HPOConstant: return self.elements[key] def __iter__(self) -> Iterator[str]: diff --git a/neps/state/neps_state.py b/neps/state/neps_state.py index cd8e88058..067620e58 100644 --- a/neps/state/neps_state.py +++ b/neps/state/neps_state.py @@ -19,6 +19,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar, overload +import numpy as np + from neps.env import ( GLOBAL_ERR_FILELOCK_POLL, GLOBAL_ERR_FILELOCK_TIMEOUT, @@ -43,6 +45,8 @@ if TYPE_CHECKING: from neps.optimizers import OptimizerInfo from neps.optimizers.optimizer import AskFunction + from neps.space import SearchSpace + from neps.space.neps_spaces.parameters import PipelineSpace from neps.utils.common import gc_disabled @@ -140,6 +144,18 @@ def _read_pkl_and_maybe_consolidate( return trials + def get_valid_evaluated_trials(self) -> dict[str, Trial]: + """Get all trials that have a valid evaluation report.""" + trials = self.latest() + return { + trial_id: trial + for trial_id, trial in trials.items() + if trial.report is not None + and trial.report.err is None + and trial.report.objective_to_minimize is not None + and not np.isnan(trial.report.objective_to_minimize) + } + def latest(self, *, create_cache_if_missing: bool = True) -> dict[str, Trial]: """Get the latest trials from the cache.""" if not self.cache_path.exists(): @@ -251,15 +267,52 @@ class NePSState: _optimizer_state_path: Path = field(repr=False) _optimizer_state: OptimizationState = field(repr=False) + _pipeline_space_path: Path = field(repr=False) + _err_lock: FileLocker = field(repr=False) _shared_errors_path: Path = field(repr=False) _shared_errors: ErrDump = field(repr=False) - new_score: float = float("inf") - """Tracking of the new incumbent""" + _pipeline_space: SearchSpace | PipelineSpace | None = field(repr=False, default=None) - all_best_configs: list = field(default_factory=list) - """Trajectory to the newest incbumbent""" + def __eq__(self, other: object) -> bool: + """Compare two NePSState objects for equality. + + Pipeline spaces are compared by pickle dumps to handle cases where + the class type differs after unpickling but the content is equivalent. + """ + if not isinstance(other, NePSState): + return NotImplemented + + # Compare all fields except _pipeline_space + for field_name in [ + "path", + "_trial_lock", + "_trial_repo", + "_optimizer_lock", + "_optimizer_info_path", + "_optimizer_info", + "_optimizer_state_path", + "_optimizer_state", + "_pipeline_space_path", + "_err_lock", + "_shared_errors_path", + "_shared_errors", + ]: + if getattr(self, field_name) != getattr(other, field_name): + return False + + # Compare pipeline spaces by pickle dumps + self_space = self._pipeline_space + other_space = other._pipeline_space + + if self_space is None and other_space is None: + return True + if self_space is None or other_space is None: + return False + + # Compare using pickle dumps - safe and handles all cases + return pickle.dumps(self_space) == pickle.dumps(other_space) def lock_and_set_new_worker_id(self, worker_id: str | None = None) -> str: """Acquire the state lock and set a new worker id in the optimizer state. @@ -288,8 +341,8 @@ def lock_and_set_new_worker_id(self, worker_id: str | None = None) -> str: ) if opt_state.worker_ids and worker_id in opt_state.worker_ids: raise NePSError( - f"Worker id '{worker_id}' already exists, \ - reserved worker ids: {opt_state.worker_ids}" + f"Worker id '{worker_id}' already exists," + f" reserved worker ids: {opt_state.worker_ids}" ) if opt_state.worker_ids is None: opt_state.worker_ids = [] @@ -335,12 +388,12 @@ def lock_and_sample_trial( return trials def lock_and_import_trials( - self, data: list, *, worker_id: str + self, imported_configs: list, *, worker_id: str ) -> Trial | list[Trial]: """Acquire the state lock and import trials from external data. Args: - data: List of trial dictionaries to import. + imported_configs: List of trial dictionaries to import. worker_id: The worker ID performing the import. Returns: @@ -351,18 +404,27 @@ def lock_and_import_trials( NePSError: If storing or reporting trials fails. """ with self._optimizer_lock.lock(), gc_disabled(): - trials = Trial.load_from_dict(data=data, worker_id=worker_id) + imported_configs = Trial.load_from_dict( + data=imported_configs, + worker_id=worker_id, + trial_directory=self._trial_repo.directory, + ) with self._trial_lock.lock(): - self._trial_repo.store_new_trial(trials) - for trial in trials: + self._trial_repo.store_new_trial(imported_configs) + for trial in imported_configs: assert trial.report is not None self._report_trial_evaluation( trial=trial, report=trial.report, worker_id=worker_id, ) - return trials + # Log imported trial similar to normal evaluation + logger.info( + f"Imported trial {trial.id} with result: " + f"{trial.report.objective_to_minimize}." + ) + return imported_configs def lock_and_report_trial_evaluation( self, @@ -538,6 +600,18 @@ def lock_and_get_optimizer_info(self) -> OptimizerInfo: with self._optimizer_lock.lock(): return _deserialize_optimizer_info(self._optimizer_info_path) + def lock_and_get_search_space(self) -> SearchSpace | PipelineSpace | None: + """Get the pipeline space, with the lock acquired. + + Returns: + The pipeline space if it was saved to disk, None otherwise. + """ + with self._optimizer_lock.lock(): + if not self._pipeline_space_path.exists(): + return None + with self._pipeline_space_path.open("rb") as f: + return pickle.load(f) # noqa: S301 + def lock_and_get_optimizer_state(self) -> OptimizationState: """Get the optimizer state.""" with self._optimizer_lock.lock(): # noqa: SIM117 @@ -622,13 +696,14 @@ def lock_and_get_current_evaluating_trials(self) -> list[Trial]: ] @classmethod - def create_or_load( + def create_or_load( # noqa: C901, PLR0912, PLR0915 cls, path: Path, *, load_only: bool = False, optimizer_info: OptimizerInfo | None = None, optimizer_state: OptimizationState | None = None, + pipeline_space: SearchSpace | PipelineSpace | None = None, ) -> NePSState: """Create a new NePSState in a directory or load the existing one if it already exists, depending on the argument. @@ -644,17 +719,23 @@ def create_or_load( In principal, we could allow multiple optimizers to be run and share the same set of trials. + We do the same check for the pipeline space, if provided. + Args: path: The directory to create the state in. load_only: If True, only load the state and do not create a new one. optimizer_info: The optimizer info to use. optimizer_state: The optimizer state to use. + pipeline_space: The pipeline space to save. Optional - if provided, it will be + saved to disk and validated on subsequent loads. Returns: The NePSState. Raises: - NePSError: If the optimizer info on disk does not match the one provided. + NePSError: If the optimizer info on disk does not match the one provided, + or if the pipeline space on disk does not match the one provided. + FileNotFoundError: If load_only=True and no NePSState exists at the path. """ path = path.absolute().resolve() is_new = not path.exists() @@ -664,6 +745,7 @@ def create_or_load( else: assert optimizer_info is not None assert optimizer_state is not None + # TODO: assert pipeline_space is None -> optional for backward compatibility path.mkdir(parents=True, exist_ok=True) config_dir = path / "configs" @@ -671,6 +753,7 @@ def create_or_load( optimizer_info_path = path / "optimizer_info.yaml" optimizer_state_path = path / "optimizer_state.pkl" + pipeline_space_path = path / "pipeline_space.pkl" shared_errors_path = path / "shared_errors.jsonl" # We have to do one bit of sanity checking to ensure that the optimzier @@ -686,12 +769,72 @@ def create_or_load( if not load_only and existing_info != optimizer_info: raise NePSError( "The optimizer info on disk does not match the one provided." - f"\nOn disk: {existing_info}\nProvided: {optimizer_info}" - f"\n\nLoaded the one on disk from {path}." + f"\nOn disk: {existing_info}" + f"\n Loaded from {path}." + f"\nProvided: {optimizer_info}" ) with optimizer_state_path.open("rb") as f: optimizer_state = pickle.load(f) # noqa: S301 + # Load and validate pipeline space if it exists + if pipeline_space_path.exists(): + try: + with pipeline_space_path.open("rb") as f: + existing_space = pickle.load(f) # noqa: S301 + except (EOFError, pickle.UnpicklingError) as e: + # File exists but is empty or corrupted (race condition during write) + # Treat as if file doesn't exist yet + logger.debug( + f"Could not load pipeline_space.pkl (possibly being written): {e}" + ) + existing_space = None + else: + if not load_only and pipeline_space is not None: + # Compare semantic attributes instead of raw pickle bytes + # This allows trivial changes like renaming the space class + from neps.space.neps_spaces.parameters import PipelineSpace as PS + + if isinstance(existing_space, PS) and isinstance( + pipeline_space, PS + ): + # Compare the actual parameter definitions + if pickle.dumps(existing_space.get_attrs()) != pickle.dumps( + pipeline_space.get_attrs() + ): + raise NePSError( + "The pipeline space parameters on disk do not match" + " those provided.\nPipeline space is saved at:" + f" {pipeline_space_path}\n\nTo continue this run:" + " either omit the pipeline_space parameter or use" + " neps.load_pipeline_space() to load the existing" + " one.\n\nTo start a new run with different" + " parameters, use a different root_directory or set" + " overwrite_root_directory=True." + ) + elif pickle.dumps(existing_space) != pickle.dumps(pipeline_space): + # Fallback for non-PipelineSpace objects (SearchSpace) + raise NePSError( + "The pipeline space on disk does not match the one" + " provided.\nPipeline space is saved at:" + f" {pipeline_space_path}\n\nTo continue this run: either" + " omit the pipeline_space parameter or use" + " neps.load_pipeline_space() to load the existing" + " one.\n\nTo start a new run with a different pipeline" + " space, use a different root_directory or set" + " overwrite_root_directory=True." + ) + pipeline_space = existing_space + elif pipeline_space is None and not load_only: + # No pipeline space on disk and none provided for a new/continued run + # This is fine for backward compatibility (old runs) but log info + logger.info( + "No pipeline space provided and none found on disk. " + "This is fine for backward compatibility but consider providing one." + ) + elif pipeline_space is None: + # load_only=True and no pipeline space on disk - fine for backward compat + pass + optimizer_info = existing_info error_dump = ReaderWriterErrDump.read(shared_errors_path) else: @@ -702,6 +845,27 @@ def create_or_load( with optimizer_state_path.open("wb") as f: pickle.dump(optimizer_state, f, protocol=pickle.HIGHEST_PROTOCOL) + # Save pipeline space if provided + if pipeline_space is not None: + with atomic_write(pipeline_space_path, "wb") as f: + pickle.dump(pipeline_space, f, protocol=pickle.HIGHEST_PROTOCOL) + + # Reload the pipeline space from disk so that the instance + # returned by `create_or_load` matches the on-disk representation + # (this ensures equality checks that compare pickled bytes + # behave consistently between subsequent loads). + try: + with pipeline_space_path.open("rb") as f: + pipeline_space = pickle.load(f) # noqa: S301 + except (EOFError, pickle.UnpicklingError, OSError) as e: + # If reloading fails for expected reasons (corrupt write, + # incomplete file due to race, or IO error) log a warning + # and fall back to the original in-memory `pipeline_space` + # so we don't break creation. We explicitly catch a + # restricted set of exceptions to avoid swallowing + # unexpected errors. + logger.warning("Reloading pipeline_space after write failed: %s", e) + error_dump = ErrDump([]) return NePSState( @@ -728,8 +892,10 @@ def create_or_load( _optimizer_info=optimizer_info, _optimizer_state_path=optimizer_state_path, _optimizer_state=optimizer_state, # type: ignore + _pipeline_space_path=pipeline_space_path, _shared_errors_path=shared_errors_path, _shared_errors=error_dump, + _pipeline_space=pipeline_space, ) @@ -739,7 +905,7 @@ def _deserialize_optimizer_info(path: Path) -> OptimizerInfo: deserialized = deserialize(path) if "name" not in deserialized or "info" not in deserialized: raise NePSError( - f"Invalid optimizer info deserialized from" + "Invalid optimizer info deserialized from" f" {path}. Did not find" " keys 'name' and 'info'." ) diff --git a/neps/state/pipeline_eval.py b/neps/state/pipeline_eval.py index 6b5f57cdd..8f70ac87e 100644 --- a/neps/state/pipeline_eval.py +++ b/neps/state/pipeline_eval.py @@ -136,7 +136,7 @@ def __post_init__(self) -> None: # noqa: C901, PLR0912 case _: raise ValueError( "The 'learning_curve' should be either a sequence of floats," - f" a sequence of sequences of floats or None." + " a sequence of sequences of floats or None." f" Got {self.learning_curve}" ) @@ -303,7 +303,7 @@ def parse( # noqa: C901, PLR0912, PLR0915 cost = float(popped_cost) case _: raise ValueError( - f"The 'cost' should be either a float or None." + "The 'cost' should be either a float or None." f" Got {popped_cost}" ) @@ -338,7 +338,7 @@ def parse( # noqa: C901, PLR0912, PLR0915 case _: raise ValueError( "The 'learning_curve' should be either a sequence of floats," - f" a sequence of sequences of floats or None." + " a sequence of sequences of floats or None." f" Got {popped_curve}" ) @@ -406,7 +406,11 @@ def _eval_trial( time_end = time.time() match user_result: case dict(): - filtered_data = {k: v for k, v in user_result.items() if k != "info_dict"} + filtered_data = { + k: v + for k, v in user_result.items() + if k not in ["info_dict", "learning_curve"] + } logger.info(f"Successful evaluation of '{trial.id}': {filtered_data}.") case _: # TODO: Revisit this and check all possible cases logger.info(f"Successful evaluation of '{trial.id}': {user_result}.") diff --git a/neps/state/trial.py b/neps/state/trial.py index 8b40e8076..34b5d9059 100644 --- a/neps/state/trial.py +++ b/neps/state/trial.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from dataclasses import dataclass from enum import Enum +from pathlib import Path from typing import Any, ClassVar, Literal from typing_extensions import Self @@ -148,11 +149,22 @@ def load_from_dict( data: list, *, worker_id: str, + trial_directory: Path, ) -> list[Self]: - """Load a trial from a dictionary with state EXTERNAL.""" - trials: list[Self] = [] + """Load a trial from a dictionary with state EXTERNAL. + + Args: + data: A list of ImportedConfig objects to load. + worker_id: The worker id that is importing the trials. + trial_directory: The directory where trials are stored. + + Returns: + A list of Trial objects. + """ + loaded_trials: list[Self] = [] for i, imported_conf in enumerate(data): info_dict = imported_conf.result.get("info_dict") or {} + location = str(trial_directory / f"config_{imported_conf.id}") trial = cls( config=imported_conf.config, @@ -163,13 +175,13 @@ def load_from_dict( time_started=info_dict.get("time_started"), time_end=info_dict.get("time_end"), evaluation_duration=info_dict.get("evaluation_duration"), - previous_trial_id=None if i == 0 else trials[i - 1].metadata.id, + previous_trial_id=( + None if i == 0 else loaded_trials[i - 1].metadata.id + ), sampling_worker_id=worker_id, evaluating_worker_id=worker_id, - location="external", - previous_trial_location=None - if i == 0 - else trials[i - 1].metadata.location, + location=location, + previous_trial_location=None, ), report=Report( reported_as="success", @@ -183,8 +195,8 @@ def load_from_dict( ), source="imported", ) - trials.append(trial) - return trials + loaded_trials.append(trial) + return loaded_trials @property def id(self) -> str: diff --git a/neps/status/status.py b/neps/status/status.py index 59447e2ba..f5ab360b3 100644 --- a/neps/status/status.py +++ b/neps/status/status.py @@ -1,19 +1,95 @@ -"""Functions to get the status of a run and save the status to CSV files.""" +"""Functions to get the status of a run and save the status to CSV files. + +This module provides utilities for monitoring NePS optimization runs. +""" # ruff: noqa: T201 from __future__ import annotations +import contextlib import itertools from collections.abc import Sequence from dataclasses import asdict, dataclass, field from pathlib import Path +from pprint import pformat +from typing import TYPE_CHECKING import numpy as np import pandas as pd +from neps.space.neps_spaces import neps_space +from neps.space.neps_spaces.neps_space import NepsCompatConverter, PipelineSpace +from neps.space.neps_spaces.sampling import OnlyPredefinedValuesSampler +from neps.space.neps_spaces.string_formatter import format_value from neps.state.neps_state import FileLocker, NePSState from neps.state.trial import State, Trial +if TYPE_CHECKING: + from neps.space.search_space import SearchSpace + + +def _format_config_entry(entry: dict, indent: str = "") -> str: + """Format a single best-config entry into a text block. + + indent is a string prefixed to the first line for nicer indentation in + the `best_config_text` block. + """ + parts: list[str] = [] + parts.append(f"{indent}Config ID: {entry['trial_id']}") + parts.append(f"Objective to minimize: {entry['score']}") + if "cost" in entry: + parts.append(f"Cost: {entry['cost']}") + + if "cumulative_evaluations" in entry: + parts.append(f"Cumulative evaluations: {entry['cumulative_evaluations']}") + if "cumulative_fidelities" in entry: + parts.append(f"Cumulative fidelities: {entry['cumulative_fidelities']}") + if "cumulative_cost" in entry: + parts.append(f"Cumulative cost: {entry['cumulative_cost']}") + if "cumulative_time" in entry: + parts.append(f"Cumulative time: {entry['cumulative_time']}") + + parts.append(f"Config: {entry['config']}") + + return "\n".join(parts) + "\n" + ("-" * 80) + "\n" + + +def _build_incumbent_content(best_configs: list[dict]) -> str: + """Build trace text and best config text from a list of best configurations. + + Args: + best_configs: List of best configuration dictionaries containing + 'trial_id', 'score', 'config', and optional metrics. + + Returns: + Tuple of (trace_text, best_config_text) strings. + """ + trace_content = ( + "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" + ) + for best in best_configs: + trace_content += _format_config_entry(best) + + return trace_content + + +def _build_optimal_set_content(best_configs: list[dict]) -> str: + """Build trace text and best config text from a list of best configurations. + + Args: + best_configs: List of best configuration dictionaries containing + 'trial_id', 'score', 'config', and optional metrics. + + Returns: + content: str. + """ + trace_text = ( + "Best configs and their objectives across evaluations:\n" + "-" * 80 + "\n" + ) + for best in best_configs: + trace_text += _format_config_entry(best) + return trace_text + @dataclass class Summary: @@ -69,12 +145,12 @@ def df(self) -> pd.DataFrame: metadata_df = pd.DataFrame.from_records( [asdict(t.metadata) for t in trials] ).convert_dtypes() - - return ( - pd.concat([config_df, extra_df, report_df, metadata_df], axis="columns") - .set_index("id") - .dropna(how="all", axis="columns") + combined_df = pd.concat( + [config_df, extra_df, report_df, metadata_df], axis="columns" ) + if combined_df.empty: + return combined_df + return combined_df.set_index("id").dropna(how="all", axis="columns") def completed(self) -> list[Trial]: """Return all trials which are in a completed state.""" @@ -100,8 +176,19 @@ def num_pending(self) -> int: """Number of trials that are pending.""" return len(self.by_state[State.PENDING]) - def formatted(self) -> str: - """Return a formatted string of the summary.""" + def formatted( # noqa: PLR0912 + self, pipeline_space: PipelineSpace | SearchSpace | None = None + ) -> str: + """Return a formatted string of the summary. + + Args: + pipeline_space: Optional PipelineSpace for the run. If provided, it is used + to format the best config in a more readable way. This is typically + auto-loaded from disk by the status() function. + + Returns: + A formatted string of the summary. + """ state_summary = "\n".join( f" {state.name.lower()}: {len(trials)}" for state, trials in self.by_state.items() @@ -115,13 +202,67 @@ def formatted(self) -> str: best_summary = "No best found yet." else: best_trial, best_objective_to_minimize = self.best + + # Format config based on whether pipeline_space_variables is provided + best_summary = ( f"# Best Found (config {best_trial.metadata.id}):" "\n" - f"\n objective_to_minimize: {best_objective_to_minimize}" - f"\n config: {best_trial.config}" - f"\n path: {best_trial.metadata.location}" + f"\n objective_to_minimize: {best_objective_to_minimize}\n config: " ) + if not pipeline_space: + # Pretty-print dict configs with proper indentation + config_str = pformat( + best_trial.config, indent=2, width=80, sort_dicts=False + ) + # Add indentation to each line for alignment + indented_config = "\n ".join(config_str.split("\n")) + best_summary += f"\n {indented_config}" + elif isinstance(pipeline_space, PipelineSpace): + # Only PipelineSpace supports pretty formatting - SearchSpace doesn't + best_config_resolve = NepsCompatConverter().from_neps_config( + best_trial.config + ) + pipeline_configs = [] + variables = list(pipeline_space.get_attrs().keys()) + list( + pipeline_space.fidelity_attrs.keys() + ) + resolved_pipeline = neps_space.resolve( + pipeline_space, + OnlyPredefinedValuesSampler(best_config_resolve.predefined_samplings), + environment_values=best_config_resolve.environment_values, + )[0] + + for variable in variables: + operation = getattr(resolved_pipeline, variable) + pipeline_configs.append(format_value(operation)) + + for n_pipeline, pipeline_config in enumerate(pipeline_configs): + formatted_config = str(pipeline_config) + variable_name = variables[n_pipeline] + + # Multi-line configs: put on new line with proper indentation + # Single-line configs: inline after variable name + if "\n" in formatted_config: + indented_config = "\n ".join( + formatted_config.split("\n") + ) + best_summary += ( + f"\n {variable_name}:\n {indented_config}" + ) + else: + best_summary += f"\n {variable_name}: {formatted_config}" + else: + # SearchSpace or other space type - pretty-print the dict + config_str = pformat( + best_trial.config, indent=2, width=80, sort_dicts=False + ) + # Add indentation to each line for alignment + indented_config = "\n ".join(config_str.split("\n")) + best_summary += f"\n {indented_config}" + + best_summary += f"\n path: {best_trial.metadata.location}" + assert best_trial.report is not None if best_trial.report.cost is not None: best_summary += f"\n cost: {best_trial.report.cost}" @@ -134,11 +275,6 @@ def formatted(self) -> str: def from_directory(cls, root_directory: str | Path) -> Summary: """Create a summary from a neps run directory.""" root_directory = Path(root_directory) - - is_multiobjective: bool = False - best: tuple[Trial, float] | None = None - by_state: dict[State, list[Trial]] = {s: [] for s in State} - # NOTE: We don't lock the shared state since we are just reading and don't need to # make decisions based on the state try: @@ -150,7 +286,20 @@ def from_directory(cls, root_directory: str | Path) -> Summary: trials = shared_state.lock_and_read_trials() - for _trial_id, trial in trials.items(): + return cls.from_trials(trials) + + @classmethod + def from_trials(cls, trials: dict[str, Trial]) -> Summary: + """Summarize a mapping of trials into (by_state, is_multiobjective, best). + + This extracts the core loop from `Summary.from_directory` so callers that + already have a `trials` mapping can reuse the logic without re-reading state. + """ + is_multiobjective: bool = False + best: tuple[Trial, float] | None = None + by_state: dict[State, list[Trial]] = {s: [] for s in State} + + for trial in trials.values(): state = trial.metadata.state by_state[state].append(trial) @@ -180,16 +329,26 @@ def status( Args: root_directory: The root directory given to neps.run. - print_summary: If true, print a summary of the current run state + print_summary: If true, print a summary of the current run state. Returns: Dataframe of full results and short summary series. """ root_directory = Path(root_directory) + + # Try to load pipeline_space from disk for pretty printing + pipeline_space = None + if print_summary: + from neps.api import load_pipeline_space + + with contextlib.suppress(FileNotFoundError, ValueError): + pipeline_space = load_pipeline_space(root_directory) + # Note: pipeline_space can still be None if it wasn't saved, which is fine + summary = Summary.from_directory(root_directory) if print_summary: - print(summary.formatted()) + print(summary.formatted(pipeline_space=pipeline_space)) df = summary.df() @@ -228,82 +387,6 @@ def status( return df, short -def trajectory_of_improvements( - root_directory: str | Path, -) -> list[dict]: - """Track and write the trajectory of improving configurations over time. - - Args: - root_directory: The root directory given to neps.run. - - Returns: - List of dicts with improving scores and their configurations. - """ - root_directory = Path(root_directory) - summary = Summary.from_directory(root_directory) - - if summary.is_multiobjective: - return [] - - df = summary.df() - - if len(df) == 0: - return [] - - if "time_sampled" not in df.columns: - raise ValueError("Missing `time_sampled` column in summary DataFrame.") - - df = df.sort_values("time_sampled") - - all_best_configs = [] - best_score = float("inf") - trace_text = "" - - for trial_id, row in df.iterrows(): - if "objective_to_minimize" not in row or pd.isna(row["objective_to_minimize"]): - continue - - score = row["objective_to_minimize"] - if score < best_score: - best_score = score - config = { - k.replace("config.", ""): v - for k, v in row.items() - if k.startswith("config.") - } - - best = { - "score": score, - "trial_id": trial_id, - "config": config, - } - all_best_configs.append(best) - - trace_text += ( - f"Objective to minimize: {best['score']}\n" - f"Config ID: {best['trial_id']}\n" - f"Config: {best['config']}\n" + "-" * 80 + "\n" - ) - - summary_dir = root_directory / "summary" - summary_dir.mkdir(parents=True, exist_ok=True) - output_path = summary_dir / "best_config_trajectory.txt" - with output_path.open("w") as f: - f.write(trace_text) - - if all_best_configs: - final_best = all_best_configs[-1] - best_path = summary_dir / "best_config.txt" - with best_path.open("w") as f: - f.write( - f"Objective to minimize: {final_best['score']}\n" - f"Config ID: {final_best['trial_id']}\n" - f"Config: {final_best['config']}\n" - ) - - return all_best_configs - - def _initiate_summary_csv(root_directory: str | Path) -> tuple[Path, Path, FileLocker]: """Initializes a summary CSV and an associated locker for file access control. diff --git a/neps/utils/__init__.py b/neps/utils/__init__.py index 7f2ee9e3f..fbc96b0eb 100644 --- a/neps/utils/__init__.py +++ b/neps/utils/__init__.py @@ -1,5 +1,7 @@ +from neps.space.neps_spaces.neps_space import convert_operation_to_callable from neps.utils.trial_io import load_trials_from_pickle __all__ = [ + "convert_operation_to_callable", "load_trials_from_pickle", ] diff --git a/neps/utils/trial_io.py b/neps/utils/trial_io.py index 8a9c948eb..6e0397a1b 100644 --- a/neps/utils/trial_io.py +++ b/neps/utils/trial_io.py @@ -5,9 +5,10 @@ from collections.abc import Mapping, Sequence, ValuesView from dataclasses import asdict from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from neps.state.neps_state import TrialRepo +from neps.state.pipeline_eval import UserResultDict if TYPE_CHECKING: from neps.state.trial import Trial @@ -15,7 +16,7 @@ def load_trials_from_pickle( root_dir: Path | str, -) -> Sequence[tuple[Mapping[str, Any], dict]]: +) -> Sequence[tuple[Mapping[str, Any], UserResultDict]]: """Load trials from a pickle-based TrialRepo. Args: @@ -37,7 +38,7 @@ def load_trials_from_pickle( ) return [ - (trial.config, asdict(trial.report)) + (trial.config, cast(UserResultDict, asdict(trial.report))) for trial in trials if trial.report is not None ] diff --git a/neps/validation.py b/neps/validation.py index 8d64cbc9a..8a50a2a76 100644 --- a/neps/validation.py +++ b/neps/validation.py @@ -3,12 +3,16 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any +from typing_extensions import assert_never from neps.exceptions import TrialValidationError +from neps.space import SearchSpace +from neps.space.neps_spaces.parameters import PipelineSpace if TYPE_CHECKING: - from neps.space import SearchSpace + from neps.space.neps_spaces.parameters import Categorical, Float, Integer + from neps.space.parameters import HPOCategorical, HPOConstant, HPOFloat, HPOInteger from neps.state.pipeline_eval import UserResultDict @@ -18,28 +22,146 @@ def _validate_imported_result(result: UserResultDict) -> None: raise TrialValidationError(config=result, message="Missing objective_to_minimize") -def _validate_imported_config( - space: SearchSpace, config: Mapping[str, float] -) -> None | Exception: +def validate_parameter_value( + param: ( + HPOFloat + | HPOInteger + | HPOCategorical + | HPOConstant + | Float + | Integer + | Categorical + ), + value: Any, +) -> bool: + """Validate a parameter value against its parameter definition. + + Works with both SearchSpace parameters (HPOFloat, HPOInteger, HPOCategorical, + HPOConstant) and PipelineSpace parameters (Float, Integer, Categorical from + neps_spaces). + + Args: + param: The parameter definition (from either SearchSpace or PipelineSpace) + value: The value to validate + + Returns: + bool: True if the value is valid for the parameter, False otherwise + """ + # Import here to avoid circular dependencies + from neps.space.neps_spaces.parameters import ( + Categorical as NepsCategorical, + Float as NepsFloat, + Integer as NepsInteger, + ) + from neps.space.parameters import ( + HPOCategorical, + HPOConstant, + HPOFloat, + HPOInteger, + ) + + # Float parameters - both use .lower and .upper + if isinstance(param, HPOFloat | NepsFloat): + return isinstance(value, float | int) and param.lower <= value <= param.upper + + # Integer parameters - both use .lower and .upper + if isinstance(param, HPOInteger | NepsInteger): + return isinstance(value, int) and param.lower <= value <= param.upper + + # Categorical parameters - both use .choices + if isinstance(param, HPOCategorical): + choices = param.choices + return value in choices + if isinstance(param, NepsCategorical): + return ( + 0 + <= value + < (len(list(param.choices)) if isinstance(param.choices, tuple) else 1) + ) + + # Constant - SearchSpace only + if isinstance(param, HPOConstant): + return value == param.value + + # Exhaustiveness check - all cases should be covered + assert_never(param) + + +def _validate_imported_config( # noqa: C901, PLR0912 + space: SearchSpace | PipelineSpace, config: Mapping[str, float] +) -> None: """Validate a configuration against the search space. Args: - space (SearchSpace): The search space to validate against. - config (dict): The configuration to validate. + space: The search space to validate against. + config: The configuration to validate. Raises: - ValueError: If the configuration is not valid. + TrialValidationError: If the configuration is not valid. """ - all_params = {**space.searchables, **space.fidelities} - for key in space.searchables: - if key not in config: - raise TrialValidationError(config=config, message=f"Missing key: {key}") - - for key, param in all_params.items(): - if key in config and not param.validate(config[key]): - raise TrialValidationError( - config=config, - message=f"Invalid value for parameter: {key}", - ) - return None + if isinstance(space, SearchSpace): + all_params = {**space.searchables, **space.fidelities} + for key in space.searchables: + if key not in config: + raise TrialValidationError(config=config, message=f"Missing key: {key}") + + for key, param in all_params.items(): + if key in config and not validate_parameter_value(param, config[key]): + raise TrialValidationError( + config=config, + message=f"Invalid value for parameter: {key}", + ) + elif isinstance(space, PipelineSpace): + # For PipelineSpace, we need to check for the prefixed keys + # Import here to avoid circular import + from neps.space.neps_spaces.neps_space import ( + NepsCompatConverter, + construct_sampling_path, + ) + from neps.space.neps_spaces.parameters import Domain + + # Check that all expected parameter keys are present in the config + for param_name, param_obj in space.get_attrs().items(): + if isinstance(param_obj, Domain): + # Construct the expected sampling path + sampling_path = construct_sampling_path( + path_parts=["Resolvable", param_name], + domain_obj=param_obj, + ) + expected_key = f"{NepsCompatConverter._SAMPLING_PREFIX}{sampling_path}" + if expected_key not in config: + raise TrialValidationError( + config=config, message=f"Missing key: {expected_key}" + ) + + # Check that all expected fidelity keys are present in the config + for fidelity_name in space.fidelity_attrs: + expected_key = f"{NepsCompatConverter._ENVIRONMENT_PREFIX}{fidelity_name}" + if expected_key not in config: + raise TrialValidationError( + config=config, message=f"Missing fidelity key: {expected_key}" + ) + + # Validate parameter values for PipelineSpace + # Note: PipelineSpace doesn't have a searchables attribute like SearchSpace + # We need to validate the attrs that are actual parameters + from neps.space.neps_spaces.parameters import Categorical, Float, Integer + + for param_name, param in space.get_attrs().items(): # type: ignore[unreachable] + if isinstance(param, Float | Integer | Categorical): # type: ignore[unreachable] + # Construct the expected sampling path and key + sampling_path = construct_sampling_path( # type: ignore[unreachable] + path_parts=["Resolvable", param_name], + domain_obj=param, + ) + expected_key = f"{NepsCompatConverter._SAMPLING_PREFIX}{sampling_path}" + + # Validate the value if the key is present in config + if expected_key in config and not validate_parameter_value( + param, config[expected_key] + ): + raise TrialValidationError( + config=config, + message=f"Invalid value for parameter: {expected_key}", + ) diff --git a/neps_examples/__init__.py b/neps_examples/__init__.py index f1c8f4631..039f85bc3 100644 --- a/neps_examples/__init__.py +++ b/neps_examples/__init__.py @@ -1,11 +1,14 @@ all_main_examples = { # Used for printing in python -m neps_examples "basic_usage": [ - "analyse", - "architecture", - "architecture_and_hyperparameters", - "hyperparameters", + "1_hyperparameters", + "2_run_analysis", + "3_architecture_search", + "4_architecture_and_hyperparameters", + "5_optimizer_search", ], "convenience": [ + "create_and_import_custom_config", + "import_trial", "logging_additional_info", "neps_tblogger_tutorial", "running_on_slurm_scripts", @@ -23,16 +26,17 @@ } core_examples = [ # Run locally and on github actions - "basic_usage/hyperparameters", # NOTE: This needs to be first for some tests to work - "basic_usage/analyse", - "experimental/expert_priors_for_architecture_and_hyperparameters", + "basic_usage/1_hyperparameters", # NOTE: This needs to be first for some tests to work + "basic_usage/2_run_analysis", + "basic_usage/3_architecture_search", + "basic_usage/4_architecture_and_hyperparameters", + "basic_usage/5_optimizer_search", "efficiency/multi_fidelity", + "efficiency/expert_priors_for_hyperparameters", + "efficiency/multi_fidelity_and_expert_priors", ] ci_examples = [ # Run on github actions - "basic_usage/architecture_and_hyperparameters", - "experimental/hierarchical_architecture", - "efficiency/expert_priors_for_hyperparameters", "convenience/logging_additional_info", "convenience/working_directory_per_pipeline", "convenience/neps_tblogger_tutorial", diff --git a/neps_examples/convenience/async_evaluation/run_pipeline.py b/neps_examples/async_evaluation/run_pipeline.py similarity index 100% rename from neps_examples/convenience/async_evaluation/run_pipeline.py rename to neps_examples/async_evaluation/run_pipeline.py diff --git a/neps_examples/convenience/async_evaluation/submit.py b/neps_examples/async_evaluation/submit.py similarity index 85% rename from neps_examples/convenience/async_evaluation/submit.py rename to neps_examples/async_evaluation/submit.py index b70e8d0f1..4d504e5bb 100644 --- a/neps_examples/convenience/async_evaluation/submit.py +++ b/neps_examples/async_evaluation/submit.py @@ -34,14 +34,13 @@ def evaluate_pipeline_via_slurm(pipeline_id, pipeline_directory, previous_pipeli return None -pipeline_space = dict( - optimizer=neps.Categorical(choices=["sgd", "adam"]), - lr=neps.Float(lower=10e-7, upper=10e-3, log=True), -) +class ExampleSpace(neps.PipelineSpace): + optimizer=neps.Categorical(choices=["sgd", "adam"]) + lr=neps.Float(lower=10e-7, upper=10e-3, log=True) neps.run( evaluate_pipeline=evaluate_pipeline_via_slurm, - pipeline_space=pipeline_space, - root_directory="results", + pipeline_space=ExampleSpace(), + root_directory="results/async_evaluation", evaluations_to_spend=2, ) diff --git a/neps_examples/basic_usage/1_hyperparameters.py b/neps_examples/basic_usage/1_hyperparameters.py new file mode 100644 index 000000000..3383aec1a --- /dev/null +++ b/neps_examples/basic_usage/1_hyperparameters.py @@ -0,0 +1,37 @@ +""" +This example demonstrates how to use NePS to optimize hyperparameters +of a pipeline. The pipeline is a simple function that takes in +five hyperparameters and returns their sum. +Neps uses the default optimizer to minimize this objective function. +""" + +import logging +import numpy as np +import neps + + +def evaluate_pipeline(float1, float2, categorical, integer1, integer2): + objective_to_minimize = -float( + np.sum([float1, float2, int(categorical), integer1, integer2]) + ) + return { + "objective_to_minimize": objective_to_minimize, + "cost": categorical, + } + + +class HPOSpace(neps.PipelineSpace): + float1 = neps.Float(lower=0, upper=1) + float2 = neps.Float(lower=-10, upper=10) + categorical = neps.Categorical(choices=(0, 1)) + integer1 = neps.Integer(lower=0, upper=1) + integer2 = neps.Integer(lower=1, upper=1000, log=True) + + +logging.basicConfig(level=logging.INFO) +neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=HPOSpace(), + root_directory="results/hyperparameters_example", + evaluations_to_spend=5, +) diff --git a/neps_examples/basic_usage/analyse.py b/neps_examples/basic_usage/2_run_analysis.py similarity index 69% rename from neps_examples/basic_usage/analyse.py rename to neps_examples/basic_usage/2_run_analysis.py index 70f4d765a..b93f90012 100644 --- a/neps_examples/basic_usage/analyse.py +++ b/neps_examples/basic_usage/2_run_analysis.py @@ -1,7 +1,5 @@ -"""How to generate a summary (neps.status) and visualizations (neps.plot) of a run. - +"""How to generate a summary (neps.status) of a run. Before running this example analysis, run the hyperparameters example with: - python -m neps_examples.basic_usage.hyperparameters """ @@ -11,10 +9,8 @@ # read-able and can be useful # 2. Printing a summary and reading in results. -# Alternatively use `python -m neps.status results/hyperparameters_example` full, summary = neps.status("results/hyperparameters_example", print_summary=True) config_id = "1" -print(full.head()) -print("") +print("\n", full.head(), "\n") print(full.loc[config_id]) diff --git a/neps_examples/basic_usage/3_architecture_search.py b/neps_examples/basic_usage/3_architecture_search.py new file mode 100644 index 000000000..5da92decb --- /dev/null +++ b/neps_examples/basic_usage/3_architecture_search.py @@ -0,0 +1,104 @@ +""" +This example demonstrates neural architecture search using NePS Spaces to define and +optimize PyTorch models. The search space consists of a 3-cell sequential architecture +where each cell contains a Conv2d layer followed by an activation function. The Conv2d +kernel size is sampled from integers in [2, 7], and the activation is chosen from +{ReLU, Sigmoid, Tanh}. Each cell independently samples its kernel size and activation, +allowing NePS to explore diverse architectural configurations and find optimal designs. + +Search Space Structure: + model: Sequential( + Cell_1: Sequential( + Conv2d(kernel_size=, ...), + + ), + Cell_2: Sequential( + Conv2d(kernel_size=, ...), + + ), + Cell_3: Sequential( + Conv2d(kernel_size=, ...), + + ) + ) +""" + +import numpy as np +import torch +import torch.nn as nn +import neps +import logging + + +# Define the NEPS space for the neural network architecture +# It reuses the same building blocks multiple times, with different sampled parameters. +class NN_Space(neps.PipelineSpace): + + # Parameters with prefixed _ are internal and will not be given to the evaluation + # function + _kernel_size = neps.Integer(2, 7) + + # Building blocks of the neural network architecture + # The convolution layer with sampled kernel size + _conv = neps.Operation( + operator=nn.Conv2d, + kwargs={ + "in_channels": 3, + "out_channels": 3, + "kernel_size": _kernel_size.resample(), + "padding": "same", + }, + ) + + # Non-linearity layer sampled from a set of choices + _nonlinearity = neps.Categorical( + choices=( + nn.ReLU(), + nn.Sigmoid(), + nn.Tanh(), + ) + ) + + # A cell consisting of a convolution followed by a non-linearity + _cell = neps.Operation( + operator=nn.Sequential, + args=( + _conv.resample(), + _nonlinearity.resample(), + ), + ) + + # The full model consisting of three cells stacked sequentially + # This will be given to the evaluation function as 'model' + model = neps.Operation( + operator=nn.Sequential, + args=( + _cell.resample(), + _cell.resample(), + _cell.resample(), + ), + ) + + +# Defining the pipeline, using the model from the NN_space space as callable +def evaluate_pipeline(model: torch.nn.Module) -> float: + x = torch.ones(size=[1, 3, 220, 220]) + result = np.sum(model(x).detach().numpy().flatten()) + + return result + + +if __name__ == "__main__": + # Run NePS with the defined pipeline and space and show the best configuration + pipeline_space = NN_Space() + logging.basicConfig(level=logging.INFO) + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=pipeline_space, + root_directory="results/architecture_search_example", + evaluations_to_spend=5, + ) + neps.status( + "results/architecture_search_example", + print_summary=True, + ) diff --git a/neps_examples/basic_usage/4_architecture_and_hyperparameters.py b/neps_examples/basic_usage/4_architecture_and_hyperparameters.py new file mode 100644 index 000000000..6ab6896f1 --- /dev/null +++ b/neps_examples/basic_usage/4_architecture_and_hyperparameters.py @@ -0,0 +1,106 @@ +""" +This example demonstrates joint optimization of neural architecture and hyperparameters +using NePS Spaces. The search space includes: (1) a 3-cell sequential architecture with +Conv2d layers (kernel size sampled from [2, 7]) and activations chosen from {ReLU, +Sigmoid, Tanh}, and (2) a batch_size hyperparameter sampled from integers in [16, 128]. +NePS simultaneously optimizes both architectural choices and training hyperparameters. + +Search Space Structure: + batch_size: + model: Sequential( + Cell_1: Sequential( + Conv2d(kernel_size=, ...), + + ), + Cell_2: Sequential( + Conv2d(kernel_size=, ...), + + ), + Cell_3: Sequential( + Conv2d(kernel_size=, ...), + + ) + ) +""" + +import neps +import torch +import numpy as np +from torch import nn +import logging + + +# Using the space from the architecture search example +class NN_Space(neps.PipelineSpace): + + # Integer Hyperparameter for the batch size + batch_size = neps.Integer(16, 128) + + # Parameters with prefixed _ are internal and will not be given to the evaluation + # function + _kernel_size = neps.Integer(2, 7) + + # Building blocks of the neural network architecture + # The convolution layer with sampled kernel size + _conv = neps.Operation( + operator=nn.Conv2d, + kwargs={ + "in_channels": 3, + "out_channels": 3, + "kernel_size": _kernel_size.resample(), + "padding": "same", + }, + ) + + # Non-linearity layer sampled from a set of choices + _nonlinearity = neps.Categorical( + choices=( + nn.ReLU(), + nn.Sigmoid(), + nn.Tanh(), + ) + ) + + # A cell consisting of a convolution followed by a non-linearity + _cell = neps.Operation( + operator=nn.Sequential, + args=( + _conv.resample(), + _nonlinearity.resample(), + ), + ) + + # The full model consisting of three cells stacked sequentially + # This will be given to the evaluation function as 'model' + model = neps.Operation( + operator=nn.Sequential, + args=( + _cell.resample(), + _cell.resample(), + _cell.resample(), + ), + ) + + +def evaluate_pipeline(model: torch.nn.Module, batch_size: int) -> float: + # For demonstration, we return a dummy objective value + # In practice, you would train and evaluate the model here + x = torch.ones(size=[1, 3, 220, 220]) + result = np.sum(model(x).detach().numpy().flatten()) + + objective_value = batch_size * result # Dummy computation + return objective_value + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=NN_Space(), + root_directory="results/architecture_with_hp_example", + evaluations_to_spend=5, + ) + neps.status( + root_directory="results/architecture_with_hp_example", + print_summary=True, + ) diff --git a/neps_examples/basic_usage/5_optimizer_search.py b/neps_examples/basic_usage/5_optimizer_search.py new file mode 100644 index 000000000..dcebea9f1 --- /dev/null +++ b/neps_examples/basic_usage/5_optimizer_search.py @@ -0,0 +1,108 @@ +""" +This example demonstrates optimizer search using NePS to design custom PyTorch optimizers +and find the best configuration. The search space defines a custom optimizer with three +gradient transformation functions sampled from {sqrt, log, exp, sign, abs}, a learning +rate sampled logarithmically from [0.0001, 0.01], and gradient clipping from [0.5, 1.0]. +NePS evaluates each optimizer by optimizing a quadratic function to discover the most +effective combination of gradient transformations and hyperparameters for convergence. + +Search Space Structure: + optimizer_class: optimizer_constructor( + , + , + , + learning_rate=, + gradient_clipping= + ) +""" + +import neps +import torch +import logging + + +def optimizer_constructor(*functions, gradient_clipping: float, learning_rate: float): + # Build a simple optimizer that applies a sequence of functions to the gradients + class CustomOptimizer(torch.optim.Optimizer): + def __init__(self, params): + defaults = dict( + gradient_clipping=gradient_clipping, learning_rate=learning_rate + ) + super().__init__(params, defaults) + + def step(self, _closure=None): + for group in self.param_groups: + for p in group["params"]: + if p.grad is None: + continue + grad = p.grad.data + for func in functions: + grad = func(grad) + # Apply gradient clipping + grad = torch.clamp( + grad, -group["gradient_clipping"], group["gradient_clipping"] + ) + # Update parameters + p.data.add_(grad, alpha=-group["learning_rate"]) + + return CustomOptimizer + + +# The search space defines the optimizer class constructed with sampled hyperparameters +# and functions +class OptimizerSpace(neps.PipelineSpace): + + # Parameters with prefixed _ are internal and will not be given to the evaluation + # function + _gradient_clipping = neps.Float(0.5, 1.0) + _learning_rate = neps.Float(0.0001, 0.01, log=True) + + _functions = neps.Categorical( + choices=(torch.sqrt, torch.log, torch.exp, torch.sign, torch.abs) + ) + + # The optimizer class constructed with the sampled functions and hyperparameters + optimizer_class = neps.Operation( + operator=optimizer_constructor, + args=( + _functions.resample(), + _functions.resample(), + _functions.resample(), + ), + kwargs={ + "learning_rate": _learning_rate.resample(), + "gradient_clipping": _gradient_clipping.resample(), + }, + ) + + +# In the pipeline, we optimize a simple quadratic function using the sampled optimizer +def evaluate_pipeline(optimizer_class) -> float: + x = torch.ones(size=[1], requires_grad=True) + optimizer = optimizer_class([x]) + + # Optimize for a few steps + for _ in range(10): + optimizer.zero_grad() + y = x**2 + 2 * x + 1 + y.backward() + optimizer.step() + + return y.item() + + +# Run NePS with the defined pipeline and space and show the best configuration +if __name__ == "__main__": + pipeline_space = OptimizerSpace() + + logging.basicConfig(level=logging.INFO) + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=pipeline_space, + root_directory="results/optimizer_search_example", + evaluations_to_spend=5, + ) + neps.status( + root_directory="results/optimizer_search_example", + print_summary=True, + ) diff --git a/neps_examples/basic_usage/example_import_trials.py b/neps_examples/basic_usage/example_import_trials.py deleted file mode 100644 index 32d9b552b..000000000 --- a/neps_examples/basic_usage/example_import_trials.py +++ /dev/null @@ -1,213 +0,0 @@ -import logging -import numpy as np -import neps -import socket -import os -from neps import UserResultDict -import random -import torch -import argparse -import neps.utils - -logging.basicConfig(level=logging.DEBUG) - -seed = 42 -random.seed(seed) -np.random.seed(seed) -torch.manual_seed(seed) -if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - -def get_evaluate_pipeline_func(optimizer): - match optimizer: - case "primo": - def evaluate_pipeline(float1, float2, categorical, integer1, integer2): - objective_to_minimize = [ - float1 - 0.3, - float2 - 3.6 - ] - return objective_to_minimize - case "ifbo": - def evaluate_pipeline(float1, float2, categorical, integer1, integer2): - objective_to_minimize = abs(float1) / abs(float( - np.sum([float1, float2, int(categorical), integer1, integer2])) - ) - return objective_to_minimize - case _: - def evaluate_pipeline(float1, float2, categorical, integer1, integer2): - objective_to_minimize = -float( - np.sum([float1, float2, int(categorical), integer1, integer2]) - ) - return objective_to_minimize - return evaluate_pipeline - - -def get_evaluated_trials(optimizer): - # Each optimizer gets its own evaluated trials fixture - match optimizer: - case "asha": - return [ - ({ - "float1": 0.5417078469603526, - "float2": 3.3333333333333335, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=-1011.5417078469603)), - ] - case "successive_halving": - return [ - ({ - "float1": 0.5417078469603526, - "float2": 3.3333333333333335, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=-1011.5417078469603)), - ] - case "priorband": - return [ - ({ - "float1": 0.5417078469603526, - "float2": 3.3333333333333335, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=-1011.5417078469603)), - ] - case "primo": - return [ - ({ - "float1": 0.5417078469603526, - "float2": 3.3333333333333335, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=[0.5417078469603, 3.3333333333333335])), - ({ - "float1": 0.5417078469603526, - "float2": 3.6, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=[0.2417078469603, 3.6])), - ] - case "ifbo": - return [ - ({ - "float1": 0.5417078469603526, - "float2": 3.3333333333333335, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=0.5417078469603)), - ({ - "float1": 0.5417078469603526, - "float2": 3.6, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=0.2417078469603)), - ] - case "hyperband": - return [ - ({ - "float1": 0.5417078469603526, - "float2": 3.3333333333333335, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=-1011.5417078469603)), - ({ - "float1": 0.5417078469603526, - "categorical": 1, - "integer1": 0, - "integer2": 800, - }, UserResultDict(objective_to_minimize=-1011.5417078469603)), - ] - case "bayesian_optimization": - return [ - ({ - "float1": 0.5884444338738143, - "float2": 3.3333333333333335, - "categorical": 0, - "integer1": 0, - "integer2": 1000, - }, {"objective_to_minimize": -1011.5417078469603}), - ] - case "async_hb": - return [ - ({ - "float1": 0.5417078469603526, - "float2": 3.3333333333333335, - "categorical": 1, - "integer1": 0, - "integer2": 1000, - }, UserResultDict(objective_to_minimize=-1011.5417078469603)), - ] - - raise ValueError(f"Unknown optimizer: {optimizer}") - -def run_import_trials(optimizer): - pipeline_space = neps.SearchSpace( - dict( - float1=neps.Float(lower=0, upper=1), - float2=neps.Float(lower=1, upper=10, is_fidelity=True), - categorical=neps.Categorical(choices=[0, 1]), - integer1=neps.Integer(lower=0, upper=1), - integer2=neps.Integer(lower=1, upper=1000, log=True), - ) - ) - - # here we write something - neps.run( - evaluate_pipeline=get_evaluate_pipeline_func(optimizer=optimizer), - pipeline_space=pipeline_space, - root_directory=f"initial_results_{optimizer}", - fidelities_to_spend=5, - worker_id=f"worker_{optimizer}-{socket.gethostname()}-{os.getpid()}", - optimizer=optimizer - ) - - trials = neps.utils.load_trials_from_pickle(root_dir=f"initial_results_{optimizer}") - - # import trials been evaluated above - neps.import_trials( - pipeline_space, - evaluated_trials=trials, - root_directory=f"results_{optimizer}", - optimizer=optimizer - ) - - # imort some trials evaluated in some other setup - neps.import_trials( - pipeline_space, - evaluated_trials=get_evaluated_trials(optimizer), - root_directory=f"results_{optimizer}", - optimizer=optimizer - ) - - neps.run( - evaluate_pipeline=get_evaluate_pipeline_func(optimizer=optimizer), - pipeline_space=pipeline_space, - root_directory=f"results_{optimizer}", - fidelities_to_spend=20, - worker_id=f"worker_{optimizer}_resume-{socket.gethostname()}-{os.getpid()}", - optimizer=optimizer - ) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--optimizer", - type=str, - required=True, - choices=[ - "asha", "successive_halving", "priorband", "primo", - "ifbo", "hyperband", "bayesian_optimization", "async_hb" - ], - help="Optimizer to test." - ) - args = parser.parse_args() - print(f"Testing import_trials for optimizer: {args.optimizer}") - run_import_trials(args.optimizer) diff --git a/neps_examples/basic_usage/hyperparameters.py b/neps_examples/basic_usage/hyperparameters.py deleted file mode 100644 index 4b9cef9dd..000000000 --- a/neps_examples/basic_usage/hyperparameters.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging -import numpy as np -import neps -import socket -import os -# This example demonstrates how to use NePS to optimize hyperparameters -# of a pipeline. The pipeline is a simple function that takes in -# five hyperparameters and returns their sum. -# Neps uses the default optimizer to minimize this objective function. - -def evaluate_pipeline(float1, float2, categorical, integer1, integer2): - objective_to_minimize = -float( - np.sum([float1, float2, int(categorical), integer1, integer2]) - ) - return {"objective_to_minimize": objective_to_minimize, "cost": categorical,} - - -pipeline_space = dict( - float1=neps.Float(lower=0, upper=1), - float2=neps.Float(lower=-10, upper=10), - categorical=neps.Categorical(choices=[0, 1]), - integer1=neps.Integer(lower=0, upper=1), - integer2=neps.Integer(lower=1, upper=1000, log=True), -) - -logging.basicConfig(level=logging.INFO) -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, - root_directory="results/hyperparameters_example", - evaluations_to_spend=15, - cost_to_spend=2, - worker_id=f"worker_1-{socket.gethostname()}-{os.getpid()}", -) diff --git a/neps_examples/convenience/create_and_import_custom_config.py b/neps_examples/convenience/create_and_import_custom_config.py new file mode 100644 index 000000000..e78407b87 --- /dev/null +++ b/neps_examples/convenience/create_and_import_custom_config.py @@ -0,0 +1,42 @@ +"""How to create a NePS configuration manually which can then be used as imported trial.""" + +import neps +from pprint import pprint +import logging + + +# This example space demonstrates all types of parameters available in NePS. +class ExampleSpace(neps.PipelineSpace): + int1 = neps.IntegerFidelity(1, 10) + float1 = neps.Float(0.0, 1.0) + cat1 = neps.Categorical(["a", "b", "c"]) + cat2 = neps.Categorical(["x", "y", float1]) + operation1 = neps.Categorical( + choices=[ + "option1", + "option2", + neps.Operation( + operator="option3", + args=(float1, cat1.resample()), + kwargs={"param1": float1.resample()}, + ), + ] + ) + + +if __name__ == "__main__": + # We create a configuration interactively and receive both + # the configuration dictionary and a dictionary of the sampled parameters. + config, pipeline = neps.create_config(ExampleSpace()) + print("Created configuration:") + pprint(config) + + logging.basicConfig(level=logging.INFO) + # The created configuration can then be used as an imported trial in NePS optimizers. + # We demonstrate this with the fictional result of objective_to_minimize = 0.5 + neps.import_trials( + evaluated_trials=[(config, neps.UserResultDict(objective_to_minimize=0.5))], + root_directory="results/created_config_example", + pipeline_space=ExampleSpace(), + overwrite_root_directory=True, + ) diff --git a/neps_examples/convenience/import_trials.py b/neps_examples/convenience/import_trials.py new file mode 100644 index 000000000..555011c31 --- /dev/null +++ b/neps_examples/convenience/import_trials.py @@ -0,0 +1,222 @@ +import logging +import numpy as np +import neps +import socket +import os +from neps import UserResultDict +import random +import torch +import argparse +import neps.utils +from typing import Any + +logging.basicConfig(level=logging.INFO) + +seed = 42 +random.seed(seed) +np.random.seed(seed) +torch.manual_seed(seed) +if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + +def get_evaluate_pipeline_func(optimizer): + match optimizer: + case "primo": + + def evaluate_pipeline_MO(float1, float2, **kwargs): + objective_to_minimize = [float1 - 0.3, float2 - 3.6] + return objective_to_minimize + + evaluate_pipeline = evaluate_pipeline_MO + + case "ifbo": + + def evaluate_pipeline_IFBO(float1, float2, categorical, integer1, integer2): + objective_to_minimize = abs(float1) / abs( + float(np.sum([float1, float2, int(categorical), integer1, integer2])) + ) + return objective_to_minimize + + evaluate_pipeline = evaluate_pipeline_IFBO + + case _: + + def evaluate_pipeline_default( + float1, float2, categorical, integer1, integer2 + ): + objective_to_minimize = -float( + np.sum([float1, float2, int(categorical), integer1, integer2]) + ) + return objective_to_minimize + + evaluate_pipeline = evaluate_pipeline_default + + return evaluate_pipeline + + +def get_evaluated_trials(optimizer) -> list[tuple[dict[str, Any], UserResultDict]]: + # Common config used by multiple optimizers + classic_base_config = { + "float1": 0.5417078469603526, + "float2": 3.3333333333333335, + "categorical": 1, + "integer1": 0, + "integer2": 1000, + } + neps_base_config = { + "ENVIRONMENT__float2": 1, + "SAMPLING__Resolvable.categorical::categorical__2": 0, + "SAMPLING__Resolvable.float1::float__0_1_False": 0.5, + "SAMPLING__Resolvable.integer1::integer__0_1_False": 1, + "SAMPLING__Resolvable.integer2::integer__1_1000_True": 5, + } + base_result = UserResultDict(objective_to_minimize=-1011.5417078469603) + + # Mapping of optimizers to their evaluated trials + trials_map = { + "asha": [(classic_base_config, base_result)], + "successive_halving": [(classic_base_config, base_result)], + "priorband": [(classic_base_config, base_result)], + "primo": [ + ( + classic_base_config, + UserResultDict( + objective_to_minimize=[0.5417078469603, 3.3333333333333335] + ), + ), + ( + {**classic_base_config, "float2": 3.6}, + UserResultDict(objective_to_minimize=[0.2417078469603, 3.6]), + ), + ], + "ifbo": [ + (classic_base_config, UserResultDict(objective_to_minimize=0.5417078469603)), + ( + {**classic_base_config, "float2": 3.6}, + UserResultDict(objective_to_minimize=0.2417078469603), + ), + ], + "hyperband": [ + (classic_base_config, base_result), + ( + { + "float1": 0.5417078469603526, + "categorical": 1, + "integer1": 0, + "integer2": 800, + }, + base_result, + ), + ], + "bayesian_optimization": [ + ( + { + "float1": 0.5884444338738143, + "float2": 3.3333333333333335, + "categorical": 0, + "integer1": 0, + "integer2": 1000, + }, + {"objective_to_minimize": -1011.5417078469603}, + ), + ], + "async_hb": [(classic_base_config, base_result)], + "neps_hyperband": [(neps_base_config, base_result)], + "neps_priorband": [(neps_base_config, base_result)], + } + + if optimizer not in trials_map: + raise ValueError(f"Unknown optimizer: {optimizer}") + + return trials_map[optimizer] + + +def run_import_trials(optimizer): + class ExampleSpace(neps.PipelineSpace): + float1 = neps.Float(lower=0, upper=1) + float2 = neps.FloatFidelity(lower=1, upper=10) + categorical = neps.Categorical(choices=[0, 1]) + integer1 = neps.Integer(lower=0, upper=1) + integer2 = neps.Integer(lower=1, upper=1000, log=True) + + logging.info( + f"{'-'*80} Running initial evaluations for optimizer {optimizer}. {'-'*80}" + ) + + # here we write something + neps.run( + evaluate_pipeline=get_evaluate_pipeline_func(optimizer=optimizer), + pipeline_space=ExampleSpace(), + root_directory=f"results/trial_import/initial_results_{optimizer}", + overwrite_root_directory=True, + fidelities_to_spend=5, + worker_id=f"worker_{optimizer}-{socket.gethostname()}-{os.getpid()}", + optimizer=optimizer, + ) + + trials = neps.utils.load_trials_from_pickle( + root_dir=f"results/trial_import/initial_results_{optimizer}" + ) + + logging.info( + f"{'-'*80} Importing {len(trials)} trials for optimizer {optimizer}. {'-'*80}" + ) + + # import trials been evaluated above + neps.import_trials( + evaluated_trials=trials, + root_directory=f"results/trial_import/results_{optimizer}", + pipeline_space=ExampleSpace(), + overwrite_root_directory=True, + optimizer=optimizer, + ) + + logging.info( + f"{'-'*80} Importing {len(get_evaluated_trials(optimizer))} trials for optimizer" + f" {optimizer}. {'-'*80}" + ) + + # import some trials evaluated in some other setup + neps.import_trials( + evaluated_trials=get_evaluated_trials(optimizer), + root_directory=f"results/trial_import/results_{optimizer}", + pipeline_space=ExampleSpace(), + optimizer=optimizer, + ) + + logging.info(f"{'-'*80} Running after import for optimizer {optimizer}. {'-'*80}") + + neps.run( + evaluate_pipeline=get_evaluate_pipeline_func(optimizer=optimizer), + pipeline_space=ExampleSpace(), + root_directory=f"results/trial_import/results_{optimizer}", + fidelities_to_spend=10, + worker_id=f"worker_{optimizer}_resume-{socket.gethostname()}-{os.getpid()}", + optimizer=optimizer, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--optimizer", + type=str, + required=True, + choices=[ + "asha", + "successive_halving", + "priorband", + "primo", + "ifbo", + "hyperband", + "bayesian_optimization", + "async_hb", + "neps_hyperband", + "neps_priorband", + ], + help="Optimizer to test.", + ) + args = parser.parse_args() + print(f"Testing import_trials for optimizer: {args.optimizer}") + run_import_trials(args.optimizer) diff --git a/neps_examples/convenience/logging_additional_info.py b/neps_examples/convenience/logging_additional_info.py index 3120c7db5..796b8db99 100644 --- a/neps_examples/convenience/logging_additional_info.py +++ b/neps_examples/convenience/logging_additional_info.py @@ -21,18 +21,18 @@ def evaluate_pipeline(float1, float2, categorical, integer1, integer2): } -pipeline_space = dict( - float1=neps.Float(lower=0, upper=1), - float2=neps.Float(lower=-10, upper=10), - categorical=neps.Categorical(choices=[0, 1]), - integer1=neps.Integer(lower=0, upper=1), - integer2=neps.Integer(lower=1, upper=1000, log=True), -) +class HPOSpace(neps.PipelineSpace): + float1 = neps.Float(lower=0, upper=1) + float2 = neps.Float(lower=-10, upper=10) + categorical = neps.Categorical(choices=(0, 1)) + integer1 = neps.Integer(lower=0, upper=1) + integer2 = neps.Integer(lower=1, upper=1000, log=True) + logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/logging_additional_info", evaluations_to_spend=5, ) diff --git a/neps_examples/convenience/neps_tblogger_tutorial.py b/neps_examples/convenience/neps_tblogger_tutorial.py index 938ca0e2c..952f82716 100644 --- a/neps_examples/convenience/neps_tblogger_tutorial.py +++ b/neps_examples/convenience/neps_tblogger_tutorial.py @@ -202,9 +202,7 @@ def training( optimizer.step() # Calculate validation objective_to_minimize using the objective_to_minimize_ev function. - validation_objective_to_minimize = objective_to_minimize_ev( - model, validation_loader - ) + validation_objective_to_minimize = objective_to_minimize_ev(model, validation_loader) return validation_objective_to_minimize @@ -212,14 +210,13 @@ def training( # Design the pipeline search spaces. -def pipeline_space() -> dict: - pipeline = dict( - lr=neps.Float(lower=1e-5, upper=1e-1, log=True), - optim=neps.Categorical(choices=["Adam", "SGD"]), - weight_decay=neps.Float(lower=1e-4, upper=1e-1, log=True), - ) +def pipeline_space() -> neps.PipelineSpace: + class HPOSpace(neps.PipelineSpace): + lr = neps.Float(lower=1e-5, upper=1e-1, log=True) + optim = neps.Categorical(choices=("Adam", "SGD")) + weight_decay = neps.Float(lower=1e-4, upper=1e-1, log=True) - return pipeline + return HPOSpace() ############################################################# @@ -229,13 +226,9 @@ def evaluate_pipeline(lr, optim, weight_decay): model = MLP() if optim == "Adam": - optimizer = torch.optim.Adam( - model.parameters(), lr=lr, weight_decay=weight_decay - ) + optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) elif optim == "SGD": - optimizer = torch.optim.SGD( - model.parameters(), lr=lr, weight_decay=weight_decay - ) + optimizer = torch.optim.SGD(model.parameters(), lr=lr, weight_decay=weight_decay) else: raise ValueError( "Optimizer choices are defined differently in the pipeline_space" diff --git a/neps_examples/convenience/running_on_slurm_scripts.py b/neps_examples/convenience/running_on_slurm_scripts.py index 26dac7082..c43ea01f6 100644 --- a/neps_examples/convenience/running_on_slurm_scripts.py +++ b/neps_examples/convenience/running_on_slurm_scripts.py @@ -50,15 +50,15 @@ def evaluate_pipeline_via_slurm( return validation_error -pipeline_space = dict( - optimizer=neps.Categorical(choices=["sgd", "adam"]), - learning_rate=neps.Float(lower=10e-7, upper=10e-3, log=True), -) +class HPOSpace(neps.PipelineSpace): + optimizer = neps.Categorical(choices=("sgd", "adam")) + learning_rate = neps.Float(lower=10e-7, upper=10e-3, log=True) + logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline_via_slurm, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/slurm_script_example", evaluations_to_spend=5, ) diff --git a/neps_examples/convenience/working_directory_per_pipeline.py b/neps_examples/convenience/working_directory_per_pipeline.py index 7b1b5ad13..115094cd4 100644 --- a/neps_examples/convenience/working_directory_per_pipeline.py +++ b/neps_examples/convenience/working_directory_per_pipeline.py @@ -18,16 +18,16 @@ def evaluate_pipeline(pipeline_directory: Path, float1, categorical, integer1): return objective_to_minimize -pipeline_space = dict( - float1=neps.Float(lower=0, upper=1), - categorical=neps.Categorical(choices=[0, 1]), - integer1=neps.Integer(lower=0, upper=1), -) +class HPOSpace(neps.PipelineSpace): + float1 = neps.Float(lower=0, upper=1) + categorical = neps.Categorical(choices=(0, 1)) + integer1 = neps.Integer(lower=0, upper=1) + logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/working_directory_per_pipeline", evaluations_to_spend=5, ) diff --git a/neps_examples/efficiency/expert_priors_for_hyperparameters.py b/neps_examples/efficiency/expert_priors_for_hyperparameters.py index 32633930b..38ea210ca 100644 --- a/neps_examples/efficiency/expert_priors_for_hyperparameters.py +++ b/neps_examples/efficiency/expert_priors_for_hyperparameters.py @@ -22,31 +22,31 @@ def evaluate_pipeline(some_float, some_integer, some_cat): # neps uses the default values and a confidence in this default value to construct a prior # that speeds up the search -pipeline_space = dict( - some_float=neps.Float( +class HPOSpace(neps.PipelineSpace): + some_float = neps.Float( lower=1, upper=1000, log=True, prior=900, prior_confidence="medium", - ), - some_integer=neps.Integer( + ) + some_integer = neps.Integer( lower=0, upper=50, prior=35, prior_confidence="low", - ), - some_cat=neps.Categorical( - choices=["a", "b", "c"], - prior="a", + ) + some_cat = neps.Categorical( + choices=("a", "b", "c"), + prior=0, prior_confidence="high", - ), -) + ) + logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/user_priors_example", evaluations_to_spend=15, ) diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index b067eac2b..b6eeccc0c 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -46,8 +46,9 @@ def get_model_and_optimizer(learning_rate): def evaluate_pipeline( pipeline_directory: Path, # The path associated with this configuration - previous_pipeline_directory: Path - | None, # The path associated with any previous config + previous_pipeline_directory: ( + Path | None + ), # The path associated with any previous config learning_rate: float, epoch: int, ) -> dict: @@ -82,17 +83,17 @@ def evaluate_pipeline( ) -pipeline_space = dict( - learning_rate=neps.Float(lower=1e-4, upper=1e0, log=True), - epoch=neps.Integer(lower=1, upper=10, is_fidelity=True), -) +class HPOSpace(neps.PipelineSpace): + learning_rate = neps.Float(lower=1e-4, upper=1e0, log=True) + epoch = neps.IntegerFidelity(lower=1, upper=10) + logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/multi_fidelity_example", # Optional: Do not start another evaluation after <=50 epochs, corresponds to cost # field above. - fidelities_to_spend=20 + fidelities_to_spend=20, ) diff --git a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py index 5e8ff2aab..05841fcac 100644 --- a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py +++ b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py @@ -6,42 +6,39 @@ # This example demonstrates NePS uses both fidelity and expert priors to # optimize hyperparameters of a pipeline. + def evaluate_pipeline(float1, float2, integer1, fidelity): objective_to_minimize = -float(np.sum([float1, float2, integer1])) / fidelity return objective_to_minimize -pipeline_space = dict( - float1=neps.Float( +class HPOSpace(neps.PipelineSpace): + float1 = neps.Float( lower=1, upper=1000, log=False, prior=600, prior_confidence="medium", - ), - float2=neps.Float( + ) + float2 = neps.Float( lower=-10, upper=10, prior=0, prior_confidence="medium", - ), - integer1=neps.Integer( + ) + integer1 = neps.Integer( lower=0, upper=50, prior=35, prior_confidence="low", - ), - fidelity=neps.Integer( - lower=1, - upper=10, - is_fidelity=True, - ), -) + ) + fidelity = neps.IntegerFidelity(lower=1, upper=10) + logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/multifidelity_priors", fidelities_to_spend=25, # For an alternate stopping method see multi_fidelity.py ) diff --git a/neps_examples/efficiency/pytorch_lightning_ddp.py b/neps_examples/efficiency/pytorch_lightning_ddp.py index d40a29cdd..07c3b970a 100644 --- a/neps_examples/efficiency/pytorch_lightning_ddp.py +++ b/neps_examples/efficiency/pytorch_lightning_ddp.py @@ -11,7 +11,8 @@ class ToyModel(nn.Module): - """ Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html """ + """Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html""" + def __init__(self): super(ToyModel, self).__init__() self.net1 = nn.Linear(10, 10) @@ -21,6 +22,7 @@ def __init__(self): def forward(self, x): return self.net2(self.relu(self.net1(x))) + class LightningModel(L.LightningModule): def __init__(self, lr): super().__init__() @@ -51,6 +53,7 @@ def test_step(self, batch, batch_idx): def configure_optimizers(self): return torch.optim.SGD(self.parameters(), lr=self.lr) + def evaluate_pipeline(lr=0.1, epoch=20): L.seed_everything(42) # Model @@ -70,35 +73,27 @@ def evaluate_pipeline(lr=0.1, epoch=20): test_dataloader = DataLoader(test_dataset, batch_size=20, shuffle=False) # Trainer with DDP Strategy - trainer = L.Trainer(gradient_clip_val=0.25, - max_epochs=epoch, - fast_dev_run=False, - strategy='ddp', - devices=NUM_GPU - ) + trainer = L.Trainer( + gradient_clip_val=0.25, + max_epochs=epoch, + fast_dev_run=False, + strategy="ddp", + devices=NUM_GPU, + ) trainer.fit(model, train_dataloader, val_dataloader) trainer.validate(model, test_dataloader) return trainer.logged_metrics["val_loss"].item() -pipeline_space = dict( - lr=neps.Float( - lower=0.001, - upper=0.1, - log=True, - prior=0.01 - ), - epoch=neps.Integer( - lower=1, - upper=3, - is_fidelity=True - ) - ) +class HPOSpace(neps.PipelineSpace): + lr = neps.Float(lower=0.001, upper=0.1, log=True, prior=0.01) + epoch = neps.IntegerFidelity(lower=1, upper=3) + logging.basicConfig(level=logging.INFO) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/pytorch_lightning_ddp", - fidelities_to_spend=5 - ) + fidelities_to_spend=5, +) diff --git a/neps_examples/efficiency/pytorch_lightning_fsdp.py b/neps_examples/efficiency/pytorch_lightning_fsdp.py index c7317bfc6..3f7afa609 100644 --- a/neps_examples/efficiency/pytorch_lightning_fsdp.py +++ b/neps_examples/efficiency/pytorch_lightning_fsdp.py @@ -56,23 +56,13 @@ def evaluate_pipeline(lr=0.1, epoch=20): logging.basicConfig(level=logging.INFO) - pipeline_space = dict( - lr=neps.Float( - lower=0.0001, - upper=0.1, - log=True, - prior=0.01 - ), - epoch=neps.Integer( - lower=1, - upper=3, - is_fidelity=True - ) - ) + class HPOSpace(neps.PipelineSpace): + lr = neps.Float(lower=0.001, upper=0.1, log=True, prior=0.01) + epoch = neps.IntegerFidelity(lower=1, upper=3) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/pytorch_lightning_fsdp", - fidelities_to_spend=5 - ) + fidelities_to_spend=5, + ) diff --git a/neps_examples/efficiency/pytorch_native_ddp.py b/neps_examples/efficiency/pytorch_native_ddp.py index 9477d6e95..9fc4741d5 100644 --- a/neps_examples/efficiency/pytorch_native_ddp.py +++ b/neps_examples/efficiency/pytorch_native_ddp.py @@ -1,4 +1,4 @@ -""" Some parts of this code are taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html +"""Some parts of this code are taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html Mind that this example does not run on Windows at the moment.""" @@ -32,8 +32,8 @@ def setup(rank, world_size): - os.environ['MASTER_ADDR'] = 'localhost' - os.environ['MASTER_PORT'] = '12355' + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12355" # initialize the process group dist.init_process_group("gloo", rank=rank, world_size=world_size) @@ -44,7 +44,8 @@ def cleanup(): class ToyModel(nn.Module): - """ Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html """ + """Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html""" + def __init__(self): super(ToyModel, self).__init__() self.net1 = nn.Linear(10, 10) @@ -56,7 +57,7 @@ def forward(self, x): def demo_basic(rank, world_size, loss_dict, learning_rate, epochs): - """ Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html (modified)""" + """Taken from https://pytorch.org/tutorials/intermediate/ddp_tutorial.html (modified)""" print(f"Running basic DDP example on rank {rank}.") setup(rank, world_size) @@ -88,28 +89,33 @@ def demo_basic(rank, world_size, loss_dict, learning_rate, epochs): def evaluate_pipeline(learning_rate, epochs): from torch.multiprocessing import Manager + world_size = NUM_GPU # Number of GPUs manager = Manager() loss_dict = manager.dict() - mp.spawn(demo_basic, - args=(world_size, loss_dict, learning_rate, epochs), - nprocs=world_size, - join=True) + mp.spawn( + demo_basic, + args=(world_size, loss_dict, learning_rate, epochs), + nprocs=world_size, + join=True, + ) loss = sum(loss_dict.values()) // world_size - return {'loss': loss} + return {"loss": loss} + +class HPOSpace(neps.PipelineSpace): + learning_rate = neps.Float(lower=10e-7, upper=10e-3, log=True) + epochs = neps.Integer(lower=1, upper=3) -pipeline_space = dict( - learning_rate=neps.Float(lower=10e-7, upper=10e-3, log=True), - epochs=neps.Integer(lower=1, upper=3) -) -if __name__ == '__main__': +if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - neps.run(evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, - root_directory="results/pytorch_ddp", - evaluations_to_spend=25) + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=HPOSpace(), + root_directory="results/pytorch_ddp", + evaluations_to_spend=25, + ) diff --git a/neps_examples/efficiency/pytorch_native_fsdp.py b/neps_examples/efficiency/pytorch_native_fsdp.py index 4b40e318a..4e441f761 100644 --- a/neps_examples/efficiency/pytorch_native_fsdp.py +++ b/neps_examples/efficiency/pytorch_native_fsdp.py @@ -24,18 +24,21 @@ size_based_auto_wrap_policy, ) -NUM_GPU = 8 # Number of GPUs to use for FSDP +NUM_GPU = 8 # Number of GPUs to use for FSDP + def setup(rank, world_size): - os.environ['MASTER_ADDR'] = 'localhost' - os.environ['MASTER_PORT'] = '12355' + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12355" # initialize the process group dist.init_process_group("nccl", rank=rank, world_size=world_size) + def cleanup(): dist.destroy_process_group() + class Net(nn.Module): def __init__(self): super(Net, self).__init__() @@ -62,6 +65,7 @@ def forward(self, x): output = F.log_softmax(x, dim=1) return output + def train(model, rank, world_size, train_loader, optimizer, epoch, sampler=None): model.train() ddp_loss = torch.zeros(2).to(rank) @@ -71,7 +75,7 @@ def train(model, rank, world_size, train_loader, optimizer, epoch, sampler=None) data, target = data.to(rank), target.to(rank) optimizer.zero_grad() output = model(data) - loss = F.nll_loss(output, target, reduction='sum') + loss = F.nll_loss(output, target, reduction="sum") loss.backward() optimizer.step() ddp_loss[0] += loss.item() @@ -79,7 +83,8 @@ def train(model, rank, world_size, train_loader, optimizer, epoch, sampler=None) dist.all_reduce(ddp_loss, op=dist.ReduceOp.SUM) if rank == 0: - print('Train Epoch: {} \tLoss: {:.6f}'.format(epoch, ddp_loss[0] / ddp_loss[1])) + print("Train Epoch: {} \tLoss: {:.6f}".format(epoch, ddp_loss[0] / ddp_loss[1])) + def test(model, rank, world_size, test_loader): model.eval() @@ -89,8 +94,12 @@ def test(model, rank, world_size, test_loader): for data, target in test_loader: data, target = data.to(rank), target.to(rank) output = model(data) - ddp_loss[0] += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss - pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + ddp_loss[0] += F.nll_loss( + output, target, reduction="sum" + ).item() # sum up batch loss + pred = output.argmax( + dim=1, keepdim=True + ) # get the index of the max log-probability ddp_loss[1] += pred.eq(target.view_as(pred)).sum().item() ddp_loss[2] += len(data) @@ -99,43 +108,45 @@ def test(model, rank, world_size, test_loader): test_loss = math.inf if rank == 0: test_loss = ddp_loss[0] / ddp_loss[2] - print('Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format( - test_loss, int(ddp_loss[1]), int(ddp_loss[2]), - 100. * ddp_loss[1] / ddp_loss[2])) + print( + "Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n".format( + test_loss, + int(ddp_loss[1]), + int(ddp_loss[2]), + 100.0 * ddp_loss[1] / ddp_loss[2], + ) + ) return test_loss + def fsdp_main(rank, world_size, test_loss_tensor, lr, epochs, save_model=False): setup(rank, world_size) - transform=transforms.Compose([ - transforms.ToTensor(), - transforms.Normalize((0.1307,), (0.3081,)) - ]) + transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] + ) - dataset1 = datasets.MNIST('./', train=True, download=True, - transform=transform) - dataset2 = datasets.MNIST('./', train=False, - transform=transform) + dataset1 = datasets.MNIST("./", train=True, download=True, transform=transform) + dataset2 = datasets.MNIST("./", train=False, transform=transform) - sampler1 = DistributedSampler(dataset1, rank=rank, num_replicas=world_size, shuffle=True) + sampler1 = DistributedSampler( + dataset1, rank=rank, num_replicas=world_size, shuffle=True + ) sampler2 = DistributedSampler(dataset2, rank=rank, num_replicas=world_size) - train_kwargs = {'batch_size': 64, 'sampler': sampler1} - test_kwargs = {'batch_size': 1000, 'sampler': sampler2} - cuda_kwargs = {'num_workers': 2, - 'pin_memory': True, - 'shuffle': False} + train_kwargs = {"batch_size": 64, "sampler": sampler1} + test_kwargs = {"batch_size": 1000, "sampler": sampler2} + cuda_kwargs = {"num_workers": 2, "pin_memory": True, "shuffle": False} train_kwargs.update(cuda_kwargs) test_kwargs.update(cuda_kwargs) - train_loader = torch.utils.data.DataLoader(dataset1,**train_kwargs) + train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) my_auto_wrap_policy = functools.partial( size_based_auto_wrap_policy, min_num_params=100 ) torch.cuda.set_device(rank) - init_start_event = torch.cuda.Event(enable_timing=True) init_end_event = torch.cuda.Event(enable_timing=True) @@ -163,7 +174,10 @@ def fsdp_main(rank, world_size, test_loss_tensor, lr, epochs, save_model=False): if rank == 0: init_end_event.synchronize() - print(f"CUDA event elapsed time: {init_start_event.elapsed_time(init_end_event) / 1000}sec") + print( + "CUDA event elapsed time:" + f" {init_start_event.elapsed_time(init_end_event) / 1000}sec" + ) if save_model: # use a barrier to make sure training is done on all ranks @@ -173,16 +187,16 @@ def fsdp_main(rank, world_size, test_loss_tensor, lr, epochs, save_model=False): torch.save(states, "mnist_cnn.pt") cleanup() + def evaluate_pipeline(lr=0.1, epoch=20): torch.manual_seed(42) test_loss_tensor = torch.zeros(1) test_loss_tensor.share_memory_() - mp.spawn(fsdp_main, - args=(NUM_GPU, test_loss_tensor, lr, epoch), - nprocs=NUM_GPU, - join=True) + mp.spawn( + fsdp_main, args=(NUM_GPU, test_loss_tensor, lr, epoch), nprocs=NUM_GPU, join=True + ) loss = test_loss_tensor.item() return loss @@ -194,23 +208,13 @@ def evaluate_pipeline(lr=0.1, epoch=20): logging.basicConfig(level=logging.INFO) - pipeline_space = dict( - lr=neps.Float( - lower=0.0001, - upper=0.1, - log=True, - prior=0.01 - ), - epoch=neps.Integer( - lower=1, - upper=3, - is_fidelity=True - ) - ) + class HPOSpace(neps.PipelineSpace): + lr = neps.Float(lower=0.0001, upper=0.1, log=True, prior=0.01) + epoch = neps.IntegerFidelity(lower=1, upper=3) neps.run( evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, + pipeline_space=HPOSpace(), root_directory="results/pytorch_fsdp", - fidelities_to_spend=20 - ) + fidelities_to_spend=20, + ) diff --git a/neps_examples/experimental/ask_and_tell_example.py b/neps_examples/experimental/ask_and_tell_example.py index 981809c1a..e1944985d 100644 --- a/neps_examples/experimental/ask_and_tell_example.py +++ b/neps_examples/experimental/ask_and_tell_example.py @@ -1,24 +1,24 @@ """ # AskAndTell Example: Custom Trial Execution with NePS -This script demonstrates how to use the `AskAndTell` interface from NePS to implement a custom trial execution workflow. -The `AskAndTell` interface provides full control over the evaluation loop, allowing you to manage how trials are executed +This script demonstrates how to use the `AskAndTell` interface from NePS to implement a custom trial execution workflow. +The `AskAndTell` interface provides full control over the evaluation loop, allowing you to manage how trials are executed and results are reported back to the optimizer. This is particularly useful when you need to handle trial execution manually. ## Aim of This File -The goal of this script is to run a **successive halving** optimization process with 3 rungs. The first rung will evaluate -9 trials in parallel. The trials are managed manually using the `AskAndTell` interface, and the SLURM scheduler is used -to execute the trials. This setup demonstrates how to efficiently manage parallel trial execution and integrate NePS +The goal of this script is to run a **successive halving** optimization process with 3 rungs. The first rung will evaluate +9 trials in parallel. The trials are managed manually using the `AskAndTell` interface, and the SLURM scheduler is used +to execute the trials. This setup demonstrates how to efficiently manage parallel trial execution and integrate NePS with external job schedulers. ## How to Use This Script 1. **Define the Search Space**: - The search space is defined using `neps.SearchSpace`. + The search space is defined using `neps.PipelineSpace`. 2. **Initialize the Optimizer**: - We use the `successive_halving` algorithm from NePS to optimize the search space. The optimizer is wrapped with + We use the `hyperband` algorithm from NePS to optimize the search space. The optimizer is wrapped with the `AskAndTell` interface to enable manual control of the evaluation loop. 3. **Submit Jobs**: @@ -26,7 +26,7 @@ - The `get_job_script` function generates a SLURM job script that executes the `train_worker` function for a given trial. 4. **Train Worker**: - - The `train_worker` function reads the trial configuration, evaluates a dummy objective function, and writes the + - The `train_worker` function reads the trial configuration, evaluates a dummy objective function, and writes the results to a JSON file. 5. **Main Loop**: @@ -50,6 +50,7 @@ This script serves as a template for implementing custom trial execution workflows with NePS. """ + import argparse import time from pathlib import Path @@ -61,6 +62,7 @@ from neps.optimizers.ask_and_tell import AskAndTell + def submit_job(pipeline_directory: Path, script: str) -> int: script_path = pipeline_directory / "submit.sh" print(f"Submitting the script {script_path} (see below): \n\n{script}") @@ -72,6 +74,7 @@ def submit_job(pipeline_directory: Path, script: str) -> int: job_id = int(output.split()[-1]) return job_id + def get_job_script(pipeline_directory, trial_file): script = f"""#!/bin/bash #SBATCH --job-name=mnist_toy @@ -82,6 +85,7 @@ def get_job_script(pipeline_directory, trial_file): """ return script + def train_worker(trial_file): trial_file = Path(trial_file) with open(trial_file) as f: @@ -89,17 +93,20 @@ def train_worker(trial_file): config = trial["config"] # Dummy objective - loss = (config["a"] - 0.5)**2 + ((config["b"] + 2)**2) / 5 + loss = (config["a"] - 0.5) ** 2 + ((config["b"] + 2) ** 2) / 5 out_file = trial_file.parent / f"result_{trial['id']}.json" with open(out_file, "w") as f: json.dump({"loss": loss}, f) + def main(parallel: int, results_dir: Path): - space = neps.SearchSpace( - {"a": neps.Integer(1, 13, is_fidelity=True), "b": neps.Float(1, 5)} - ) - opt = neps.algorithms.successive_halving(space, eta=3) + class MySpace(neps.PipelineSpace): + a = neps.IntegerFidelity(1, 13) + b = neps.Float(1, 5) + + space = MySpace() + opt = neps.algorithms.neps_hyperband(space, eta=3) ask_tell = AskAndTell(opt) results_dir.mkdir(exist_ok=True, parents=True) @@ -127,20 +134,30 @@ def main(parallel: int, results_dir: Path): new_trial = ask_tell.ask() if new_trial: new_file = results_dir / f"trial_{new_trial.id}.json" - json.dump({"id": new_trial.id, "config": new_trial.config}, new_file.open("w")) - new_job_id = submit_job(results_dir, get_job_script(results_dir, new_file)) + json.dump( + {"id": new_trial.id, "config": new_trial.config}, + new_file.open("w"), + ) + new_job_id = submit_job( + results_dir, get_job_script(results_dir, new_file) + ) active[new_job_id] = new_trial time.sleep(5) + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( - "--parallel", type=int, default=9, - help="Number of trials to evaluate in parallel initially" + "--parallel", + type=int, + default=9, + help="Number of trials to evaluate in parallel initially", ) parser.add_argument( - "--results-dir", type=Path, default=Path("results"), - help="Path to save the results inside" + "--results-dir", + type=Path, + default=Path("results/ask_and_tell"), + help="Path to save the results inside", ) args = parser.parse_args() main(args.parallel, args.results_dir) diff --git a/neps_examples/experimental/freeze_thaw.py b/neps_examples/experimental/freeze_thaw.py index 656927f26..fe490c1ce 100644 --- a/neps_examples/experimental/freeze_thaw.py +++ b/neps_examples/experimental/freeze_thaw.py @@ -53,14 +53,10 @@ def training_pipeline( KeyError: If the specified optimizer is not supported. """ # Transformations applied on each image - transform = transforms.Compose( - [ - transforms.ToTensor(), - transforms.Normalize( - (0.1307,), (0.3081,) - ), # Mean and Std Deviation for MNIST - ] - ) + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), # Mean and Std Deviation for MNIST + ]) # Loading MNIST dataset dataset = datasets.MNIST( @@ -83,8 +79,7 @@ def training_pipeline( if previous_pipeline_directory is not None: if (Path(previous_pipeline_directory) / "checkpoint.pt").exists(): states = torch.load( - Path(previous_pipeline_directory) / "checkpoint.pt", - weights_only=False + Path(previous_pipeline_directory) / "checkpoint.pt", weights_only=False ) model = states["model"] optimizer = states["optimizer"] @@ -153,21 +148,20 @@ def training_pipeline( if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - pipeline_space = { - "learning_rate": neps.Float(1e-5, 1e-1, log=True), - "num_layers": neps.Integer(1, 5), - "num_neurons": neps.Integer(64, 128), - "weight_decay": neps.Float(1e-5, 0.1, log=True), - "epochs": neps.Integer(1, 10, is_fidelity=True), - } + class ModelSpace(neps.PipelineSpace): + learning_rate = neps.Float(1e-5, 1e-1, log=True) + num_layers = neps.Integer(1, 5) + num_neurons = neps.Integer(64, 128) + weight_decay = neps.Float(1e-5, 0.1, log=True) + epochs = neps.IntegerFidelity(1, 10) neps.run( - pipeline_space=pipeline_space, + pipeline_space=ModelSpace(), evaluate_pipeline=training_pipeline, optimizer="ifbo", fidelities_to_spend=50, root_directory="./results/ifbo-mnist/", - overwrite_working_directory=False, # set to False for a multi-worker run + overwrite_root_directory=False, # set to False for a multi-worker run ) # NOTE: this is `experimental` and may not work as expected diff --git a/neps_examples/real_world/image_segmentation_hpo.py b/neps_examples/real_world/image_segmentation_hpo.py index 51ff27a06..67dd933cf 100644 --- a/neps_examples/real_world/image_segmentation_hpo.py +++ b/neps_examples/real_world/image_segmentation_hpo.py @@ -21,27 +21,33 @@ def __init__(self, iters_per_epoch, lr, momentum, weight_decay): def training_step(self, batch): images, targets = batch - outputs = self.model(images)['out'] + outputs = self.model(images)["out"] loss = self.loss_fn(outputs, targets.long().squeeze(1)) self.log("train_loss", loss, sync_dist=True) return loss def validation_step(self, batch): images, targets = batch - outputs = self.model(images)['out'] + outputs = self.model(images)["out"] loss = self.loss_fn(outputs, targets.long().squeeze(1)) self.log("val_loss", loss, sync_dist=True) return loss def configure_optimizers(self): - optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr, momentum=self.momentum, weight_decay=self.weight_decay) + optimizer = torch.optim.SGD( + self.model.parameters(), + lr=self.lr, + momentum=self.momentum, + weight_decay=self.weight_decay, + ) scheduler = PolynomialLR( - optimizer, total_iters=self.iters_per_epoch * self.trainer.max_epochs, power=0.9 + optimizer, + total_iters=self.iters_per_epoch * self.trainer.max_epochs, + power=0.9, ) return [optimizer], [scheduler] - class SegmentationData(L.LightningDataModule): def __init__(self, batch_size=4): super().__init__() @@ -56,29 +62,62 @@ def train_dataloader(self): transform = transforms.Compose([ transforms.ToTensor(), transforms.Resize((256, 256), antialias=True), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) - target_transform = transforms.Compose([transforms.ToTensor(), transforms.Resize((256, 256), antialias=True)]) - train_dataset = datasets.VOCSegmentation(root=".data/VOC", transform=transform, target_transform=target_transform) - return torch.utils.data.DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=16, persistent_workers=True) + target_transform = transforms.Compose( + [transforms.ToTensor(), transforms.Resize((256, 256), antialias=True)] + ) + train_dataset = datasets.VOCSegmentation( + root=".data/VOC", transform=transform, target_transform=target_transform + ) + return torch.utils.data.DataLoader( + train_dataset, + batch_size=self.batch_size, + shuffle=True, + num_workers=16, + persistent_workers=True, + ) def val_dataloader(self): transform = transforms.Compose([ transforms.ToTensor(), transforms.Resize((256, 256), antialias=True), - transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) - target_transform = transforms.Compose([transforms.ToTensor(), transforms.Resize((256, 256), antialias=True)]) - val_dataset = datasets.VOCSegmentation(root=".data/VOC", year='2012', image_set='val', transform=transform, target_transform=target_transform) - return torch.utils.data.DataLoader(val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=16, persistent_workers=True) + target_transform = transforms.Compose( + [transforms.ToTensor(), transforms.Resize((256, 256), antialias=True)] + ) + val_dataset = datasets.VOCSegmentation( + root=".data/VOC", + year="2012", + image_set="val", + transform=transform, + target_transform=target_transform, + ) + return torch.utils.data.DataLoader( + val_dataset, + batch_size=self.batch_size, + shuffle=False, + num_workers=16, + persistent_workers=True, + ) def evaluate_pipeline(**kwargs): data = SegmentationData(kwargs.get("batch_size", 4)) data.prepare_data() iters_per_epoch = len(data.train_dataloader()) - model = LitSegmentation(iters_per_epoch, kwargs.get("lr", 0.02), kwargs.get("momentum", 0.9), kwargs.get("weight_decay", 1e-4)) - trainer = L.Trainer(max_epochs=kwargs.get("epoch", 30), strategy=DDPStrategy(find_unused_parameters=True), enable_checkpointing=False) + model = LitSegmentation( + iters_per_epoch, + kwargs.get("lr", 0.02), + kwargs.get("momentum", 0.9), + kwargs.get("weight_decay", 1e-4), + ) + trainer = L.Trainer( + max_epochs=kwargs.get("epoch", 30), + strategy=DDPStrategy(find_unused_parameters=True), + enable_checkpointing=False, + ) trainer.fit(model, data) val_loss = trainer.logged_metrics["val_loss"].detach().item() return val_loss @@ -92,33 +131,11 @@ def evaluate_pipeline(**kwargs): # Search space for hyperparameters pipeline_space = dict( - lr=neps.Float( - lower=0.0001, - upper=0.1, - log=True, - prior=0.02 - ), - momentum=neps.Float( - lower=0.1, - upper=0.9, - prior=0.5 - ), - weight_decay=neps.Float( - lower=1e-5, - upper=1e-3, - log=True, - prior=1e-4 - ), - epoch=neps.Integer( - lower=10, - upper=30, - is_fidelity=True - ), - batch_size=neps.Integer( - lower=4, - upper=12, - prior=4 - ), + lr=neps.HPOFloat(lower=0.0001, upper=0.1, log=True, prior=0.02), + momentum=neps.HPOFloat(lower=0.1, upper=0.9, prior=0.5), + weight_decay=neps.HPOFloat(lower=1e-5, upper=1e-3, log=True, prior=1e-4), + epoch=neps.HPOInteger(lower=10, upper=30, is_fidelity=True), + batch_size=neps.HPOInteger(lower=4, upper=12, prior=4), ) neps.run( diff --git a/pyproject.toml b/pyproject.toml index 22f5c8341..91dcf3402 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ dev = [ "mypy>=1,<2", "pytest>=7,<8", "pytest-cases>=3,<4", + "pytest-repeat>=0,<1", "types-PyYAML>=6,<7", "mkdocs-material", "mkdocs-autorefs", @@ -241,6 +242,10 @@ ignore = [ "PT011", # Catch value error to broad "ARG001", # unused param ] +"tests/test_neps_space/*.py" = [ + "E501", # Line length for architecture strings +] + "__init__.py" = ["I002"] "neps_examples/*" = [ "INP001", @@ -255,6 +260,10 @@ ignore = [ "E501", ] "docs/*" = ["INP001"] +"neps/space/neps_spaces/sampling.py" = [ + "T201", # print() is intentional for IOSampler user interaction + "RET508", # else after break is needed for error messages +] # TODO "neps/optimizers/**.py" = [ "D", # Documentation of everything diff --git a/tests/test_config_encoder.py b/tests/test_config_encoder.py index db4a5cee6..276bc566e 100644 --- a/tests/test_config_encoder.py +++ b/tests/test_config_encoder.py @@ -2,14 +2,14 @@ import torch -from neps.space import Categorical, ConfigEncoder, Float, Integer +from neps.space import ConfigEncoder, HPOCategorical, HPOFloat, HPOInteger def test_config_encoder_pdist_calculation() -> None: parameters = { - "a": Categorical(["cat", "mouse", "dog"]), - "b": Integer(1, 10), - "c": Float(1, 10), + "a": HPOCategorical(["cat", "mouse", "dog"]), + "b": HPOInteger(1, 10), + "c": HPOFloat(1, 10), } encoder = ConfigEncoder.from_parameters(parameters) config1 = {"a": "cat", "b": 1, "c": 1.0} @@ -43,9 +43,9 @@ def test_config_encoder_pdist_calculation() -> None: def test_config_encoder_pdist_squareform() -> None: parameters = { - "a": Categorical(["cat", "mouse", "dog"]), - "b": Integer(1, 10), - "c": Float(1, 10), + "a": HPOCategorical(["cat", "mouse", "dog"]), + "b": HPOInteger(1, 10), + "c": HPOFloat(1, 10), } encoder = ConfigEncoder.from_parameters(parameters) config1 = {"a": "cat", "b": 1, "c": 1.0} diff --git a/tests/test_examples.py b/tests/test_examples.py index 5510c084e..93ed60227 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -35,19 +35,12 @@ def no_logs_gte_error(caplog): @pytest.mark.core_examples @pytest.mark.parametrize("example", core_examples_scripts, ids=core_examples) def test_core_examples(example): - if example.name == "analyse.py": + if example.name == "run_analysis.py": # Run hyperparameters example to have something to analyse runpy.run_path(str(core_examples_scripts[0]), run_name="__main__") - if example.name in ( - "architecture.py", - "architecture_and_hyperparameters.py", - "hierarchical_architecture.py", - "expert_priors_for_architecture_and_hyperparameters.py", - ): - pytest.xfail("Architecture were removed temporarily") - - runpy.run_path(str(example), run_name="__main__") + else: + runpy.run_path(str(example), run_name="__main__") @pytest.mark.ci_examples diff --git a/tests/test_neps_space/__init__.py b/tests/test_neps_space/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_neps_space/test_backward_compatibility.py b/tests/test_neps_space/test_backward_compatibility.py new file mode 100644 index 000000000..97bf5c14c --- /dev/null +++ b/tests/test_neps_space/test_backward_compatibility.py @@ -0,0 +1,302 @@ +"""Test backward compatibility with old SearchSpace and dict-based spaces.""" + +from __future__ import annotations + +import tempfile +import warnings +from pathlib import Path + +import pytest + +import neps +from neps.optimizers import algorithms +from neps.space import HPOCategorical, HPOFloat, HPOInteger, SearchSpace +from neps.space.neps_spaces.parameters import ( + Categorical, + Float, + Integer, + PipelineSpace, +) + + +def simple_evaluation(learning_rate: float, num_layers: int, optimizer: str) -> float: + """Simple evaluation function.""" + return learning_rate * num_layers + (0.1 if optimizer == "adam" else 0.2) + + +def test_searchspace_with_hpo_parameters(): + """Test SearchSpace with old HPO* parameters still works.""" + pipeline_space = SearchSpace( + { + "learning_rate": HPOFloat(1e-4, 1e-1, log=True), + "num_layers": HPOInteger(1, 10), + "optimizer": HPOCategorical(["adam", "sgd", "rmsprop"]), + } + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "searchspace_hpo_test" + + # Should warn about using SearchSpace instead of PipelineSpace + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=pipeline_space, + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Should get deprecation warning about SearchSpace + assert any( + issubclass(warning.category, DeprecationWarning) + and "SearchSpace" in str(warning.message) + for warning in w + ), "Should warn about using SearchSpace" + + assert root_directory.exists() + + +def test_searchspace_with_new_parameters(): + """Test SearchSpace with new PipelineSpace parameters (Float, Integer, Categorical).""" + pipeline_space = SearchSpace( + { + "learning_rate": Float(1e-4, 1e-1, log=True), + "num_layers": Integer(1, 10), + "optimizer": Categorical(["adam", "sgd", "rmsprop"]), + } + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "searchspace_new_test" + + # Should warn about using SearchSpace instead of PipelineSpace + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=pipeline_space, + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Should get deprecation warning about SearchSpace + assert any( + issubclass(warning.category, DeprecationWarning) + and "SearchSpace" in str(warning.message) + for warning in w + ), "Should warn about using SearchSpace" + + assert root_directory.exists() + + +def test_dict_with_hpo_parameters(): + """Test dict-based space with old HPO* parameters still works.""" + pipeline_space = { + "learning_rate": HPOFloat(1e-4, 1e-1, log=True), + "num_layers": HPOInteger(1, 10), + "optimizer": HPOCategorical(["adam", "sgd", "rmsprop"]), + } + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "dict_hpo_test" + + # Should warn about using dict instead of PipelineSpace + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=pipeline_space, + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Should get deprecation warning about dict + assert any( + issubclass(warning.category, DeprecationWarning) + and "dictionary" in str(warning.message).lower() + for warning in w + ), "Should warn about using dict" + + assert root_directory.exists() + + +def test_dict_with_new_parameters(): + """Test dict-based space with new PipelineSpace parameters (Float, Integer, Categorical).""" + pipeline_space = { + "learning_rate": Float(1e-4, 1e-1, log=True), + "num_layers": Integer(1, 10), + "optimizer": Categorical(["adam", "sgd", "rmsprop"]), + } + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "dict_new_test" + + # Should warn about using dict instead of PipelineSpace + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=pipeline_space, + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Should get deprecation warning about dict + assert any( + issubclass(warning.category, DeprecationWarning) + and "dictionary" in str(warning.message).lower() + for warning in w + ), "Should warn about using dict" + + assert root_directory.exists() + + +def test_searchspace_with_is_fidelity(): + """Test SearchSpace with is_fidelity parameter (old style) still works.""" + pipeline_space = SearchSpace( + { + "learning_rate": Float(1e-4, 1e-1, log=True), + "num_layers": Integer(1, 10), + "optimizer": Categorical(["adam", "sgd", "rmsprop"]), + "epochs": Integer(1, 100, is_fidelity=True), + } + ) + + def fidelity_evaluation( + learning_rate: float, + num_layers: int, + optimizer: str, + epochs: int, + ) -> float: + """Evaluation with fidelity.""" + return learning_rate * num_layers * epochs / 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "searchspace_fidelity_test" + + # Should work without errors (just warn about SearchSpace) + # Use neps_hyperband which supports fidelities + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + neps.run( + evaluate_pipeline=fidelity_evaluation, + pipeline_space=pipeline_space, + optimizer=algorithms.neps_hyperband, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Should get deprecation warning about SearchSpace + assert any( + issubclass(warning.category, DeprecationWarning) + and "SearchSpace" in str(warning.message) + for warning in w + ), "Should warn about using SearchSpace" + + assert root_directory.exists() + + +def test_dict_with_is_fidelity(): + """Test dict-based space with is_fidelity parameter (old style) still works.""" + pipeline_space = { + "learning_rate": Float(1e-4, 1e-1, log=True), + "num_layers": Integer(1, 10), + "optimizer": Categorical(["adam", "sgd", "rmsprop"]), + "epochs": Integer(1, 100, is_fidelity=True), + } + + def fidelity_evaluation( + learning_rate: float, + num_layers: int, + optimizer: str, + epochs: int, + ) -> float: + """Evaluation with fidelity.""" + return learning_rate * num_layers * epochs / 100 + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "dict_fidelity_test" + + # Should work without errors (just warn about dict) + # Use neps_hyperband which supports fidelities + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + neps.run( + evaluate_pipeline=fidelity_evaluation, + pipeline_space=pipeline_space, + optimizer=algorithms.neps_hyperband, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Should get deprecation warning about dict + assert any( + issubclass(warning.category, DeprecationWarning) + and "dictionary" in str(warning.message).lower() + for warning in w + ), "Should warn about using dict" + + assert root_directory.exists() + + +def test_proper_pipelinespace_no_warnings(): + """Test that using proper PipelineSpace class doesn't trigger warnings.""" + + class TestSpace(PipelineSpace): + learning_rate = Float(1e-4, 1e-1, log=True) + num_layers = Integer(1, 10) + optimizer = Categorical(["adam", "sgd", "rmsprop"]) + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "pipelinespace_test" + + # Should NOT warn when using proper PipelineSpace + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=TestSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Should NOT get deprecation warning about SearchSpace or dict + deprecation_warnings = [ + warning + for warning in w + if issubclass(warning.category, DeprecationWarning) + and ( + "SearchSpace" in str(warning.message) + or "dictionary" in str(warning.message).lower() + ) + ] + assert len(deprecation_warnings) == 0, ( + "Should not warn when using proper PipelineSpace, " + f"but got: {[str(w.message) for w in deprecation_warnings]}" + ) + + assert root_directory.exists() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_neps_space/test_basic_functionality.py b/tests/test_neps_space/test_basic_functionality.py new file mode 100644 index 000000000..1083196d4 --- /dev/null +++ b/tests/test_neps_space/test_basic_functionality.py @@ -0,0 +1,163 @@ +"""Simplified tests for basic NePS functionality.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +import pytest + +import neps +from neps.optimizers import algorithms +from neps.space.neps_spaces.parameters import ( + Float, + Integer, + PipelineSpace, +) + + +class SimpleSpace(PipelineSpace): + """Simple space for testing.""" + + x = Float(lower=0.0, upper=1.0) + y = Integer(lower=1, upper=10) + + +def simple_evaluation(x: float, y: int) -> float: + """Simple evaluation function.""" + return x + y + + +def test_basic_neps_run(): + """Test that basic NePS run functionality works.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "basic_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Check that optimization ran and created some files + assert root_directory.exists() + + # Should have created some evaluation files + files = list(root_directory.rglob("*")) + assert len(files) > 0, "Should have created some files" + + +def test_neps_optimization_with_dict_return(): + """Test NePS optimization with evaluation function returning dict.""" + + def dict_evaluation(x: float, y: int) -> dict: + return { + "objective_to_minimize": x + y, + "additional_metric": x * y, + } + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "dict_test" + + # Run optimization + neps.run( + evaluate_pipeline=dict_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Check that optimization completed + assert root_directory.exists() + + +def test_different_neps_optimizers(): + """Test that different NePS optimizers work.""" + optimizers_to_test = [ + algorithms.neps_random_search, + algorithms.complex_random_search, + ] + + for optimizer in optimizers_to_test: + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / f"optimizer_{optimizer.__name__}" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=optimizer, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Check that optimization completed + assert root_directory.exists() + + +def test_neps_status_functionality(): + """Test that neps.status works after optimization.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "status_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=5, + overwrite_root_directory=True, + ) + + # Test status functionality (should not raise an error) + try: + neps.status(str(root_directory)) + except (FileNotFoundError, ValueError, KeyError) as e: + pytest.fail(f"neps.status should work after optimization: {e}") + + +def test_evaluation_results_are_recorded(): + """Test that evaluation results are properly recorded.""" + # Track evaluations + evaluations_called = [] + + def tracking_evaluation(x: float, y: int) -> float: + result = x + y + evaluations_called.append((x, y, result)) + return result + + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "tracking_test" + + # Run optimization + neps.run( + evaluate_pipeline=tracking_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Check that evaluations were called + assert len(evaluations_called) == 3, ( + f"Expected 3 evaluations, got {len(evaluations_called)}" + ) + + # Check that all results are reasonable + for x, y, result in evaluations_called: + assert 0.0 <= x <= 1.0, f"x should be in [0,1], got {x}" + assert 1 <= y <= 10, f"y should be in [1,10], got {y}" + assert result == x + y, f"Result should be x+y, got {result} != {x}+{y}" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_neps_space/test_domain__centering.py b/tests/test_neps_space/test_domain__centering.py new file mode 100644 index 000000000..2d02a120b --- /dev/null +++ b/tests/test_neps_space/test_domain__centering.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +import pytest + +from neps.space.neps_spaces.parameters import Categorical, ConfidenceLevel, Float, Integer + + +@pytest.mark.parametrize( + ("confidence_level", "expected_prior_min_max"), + [ + (ConfidenceLevel.LOW, (50, 10, 90)), + (ConfidenceLevel.MEDIUM, (50, 25, 75)), + (ConfidenceLevel.HIGH, (50, 40, 60)), + ], +) +def test_centering_integer( + confidence_level, + expected_prior_min_max, +): + # Construct domains manually and then with priors. + # They are constructed in a way that after centering they both + # refer to identical domain ranges. + + int_prior = 50 + + int1 = Integer( + lower=1, + upper=100, + ) + int2 = Integer( + lower=1, + upper=100, + prior=int_prior, + prior_confidence=confidence_level, + ) + + int1_centered = int1.centered_around(int_prior, confidence_level) + int2_centered = int2.centered_around(int2.prior, int2.prior_confidence) + + assert int_prior == expected_prior_min_max[0] + assert ( + ( + int1_centered.prior, + int1_centered.lower, + int1_centered.upper, + ) + == ( + int2_centered.prior, + int2_centered.lower, + int2_centered.upper, + ) + == expected_prior_min_max + ) + + int1_centered.sample() + int2_centered.sample() + + +@pytest.mark.parametrize( + ("confidence_level", "expected_prior_min_max"), + [ + ( + ConfidenceLevel.LOW, + (50.0, 10.399999999999999, 89.6), + ), + (ConfidenceLevel.MEDIUM, (50.0, 25.25, 74.75)), + (ConfidenceLevel.HIGH, (50.0, 40.1, 59.9)), + ], +) +def test_centering_float( + confidence_level, + expected_prior_min_max, +): + # Construct domains manually and then with priors. + # They are constructed in a way that after centering they both + # refer to identical domain ranges. + + float_prior = 50.0 + + float1 = Float( + lower=1.0, + upper=100.0, + ) + float2 = Float( + lower=1.0, + upper=100.0, + prior=float_prior, + prior_confidence=confidence_level, + ) + + float1_centered = float1.centered_around(float_prior, confidence_level) + float2_centered = float2.centered_around(float2.prior, float2.prior_confidence) + + assert float_prior == expected_prior_min_max[0] + assert ( + ( + float1_centered.prior, + float1_centered.lower, + float1_centered.upper, + ) + == ( + float2_centered.prior, + float2_centered.lower, + float2_centered.upper, + ) + == expected_prior_min_max + ) + + float1_centered.sample() + float2_centered.sample() + + +@pytest.mark.parametrize( + ("confidence_level", "expected_prior_min_upper"), + [ + (ConfidenceLevel.LOW, (40, 0, 80, 50)), + (ConfidenceLevel.MEDIUM, (25, 0, 50, 50)), + (ConfidenceLevel.HIGH, (10, 0, 20, 50)), + ], +) +def test_centering_categorical( + confidence_level, + expected_prior_min_upper, +): + # Construct domains manually and then with priors. + # They are constructed in a way that after centering they both + # refer to identical domain ranges. + + categorical_prior_index_original = 49 + + categorical1 = Categorical( + choices=tuple(range(1, 101)), + ) + categorical2 = Categorical( + choices=tuple(range(1, 101)), + prior=categorical_prior_index_original, + prior_confidence=confidence_level, + ) + + categorical1_centered = categorical1.centered_around( + categorical_prior_index_original, confidence_level + ) + categorical2_centered = categorical2.centered_around( + categorical2.prior, categorical2.prior_confidence + ) + + # During the centering of categorical objects, the prior index will change. + assert categorical_prior_index_original != expected_prior_min_upper[0] + + assert ( + ( + categorical1_centered.prior, + categorical1_centered.lower, + categorical1_centered.upper, + categorical1_centered.choices[categorical1_centered.prior], + ) + == ( + categorical2_centered.prior, + categorical2_centered.lower, + categorical2_centered.upper, + categorical2_centered.choices[categorical2_centered.prior], + ) + == expected_prior_min_upper + ) + + categorical1_centered.sample() + categorical2_centered.sample() + + +@pytest.mark.parametrize( + ("confidence_level", "expected_prior_min_max"), + [ + (ConfidenceLevel.LOW, (10, 5, 13)), + (ConfidenceLevel.MEDIUM, (10, 7, 13)), + (ConfidenceLevel.HIGH, (10, 8, 12)), + ], +) +def test_centering_stranger_ranges_integer( + confidence_level, + expected_prior_min_max, +): + int1 = Integer( + lower=1, + upper=13, + ) + int1_centered = int1.centered_around(10, confidence_level) + + int2 = Integer( + lower=1, + upper=13, + prior=10, + prior_confidence=confidence_level, + ) + int2_centered = int2.centered_around(int2.prior, int2.prior_confidence) + + assert ( + int1_centered.prior, + int1_centered.lower, + int1_centered.upper, + ) == expected_prior_min_max + assert ( + int2_centered.prior, + int2_centered.lower, + int2_centered.upper, + ) == expected_prior_min_max + + int1_centered.sample() + int2_centered.sample() + + +@pytest.mark.parametrize( + ("confidence_level", "expected_prior_min_max"), + [ + ( + ConfidenceLevel.LOW, + (0.5, 0.09999999999999998, 0.9), + ), + (ConfidenceLevel.MEDIUM, (0.5, 0.25, 0.75)), + (ConfidenceLevel.HIGH, (0.5, 0.4, 0.6)), + ], +) +def test_centering_stranger_ranges_float( + confidence_level, + expected_prior_min_max, +): + float1 = Float( + lower=0.0, + upper=1.0, + ) + float1_centered = float1.centered_around(0.5, confidence_level) + + float2 = Float( + lower=0.0, + upper=1.0, + prior=0.5, + prior_confidence=confidence_level, + ) + float2_centered = float2.centered_around(float2.prior, float2.prior_confidence) + + assert ( + float1_centered.prior, + float1_centered.lower, + float1_centered.upper, + ) == expected_prior_min_max + assert ( + float2_centered.prior, + float2_centered.lower, + float2_centered.upper, + ) == expected_prior_min_max + + float1_centered.sample() + float2_centered.sample() + + +@pytest.mark.parametrize( + ("confidence_level", "expected_prior_min_upper"), + [ + (ConfidenceLevel.LOW, (2, 0, 5, 2)), + (ConfidenceLevel.MEDIUM, (2, 0, 4, 2)), + (ConfidenceLevel.HIGH, (1, 0, 2, 2)), + ], +) +def test_centering_stranger_ranges_categorical( + confidence_level, + expected_prior_min_upper, +): + categorical1 = Categorical( + choices=tuple(range(7)), + ) + categorical1_centered = categorical1.centered_around(2, confidence_level) + + categorical2 = Categorical( + choices=tuple(range(7)), + prior=2, + prior_confidence=confidence_level, + ) + categorical2_centered = categorical2.centered_around( + categorical2.prior, categorical2.prior_confidence + ) + + assert ( + categorical1_centered.prior, + categorical1_centered.lower, + categorical1_centered.upper, + categorical1_centered.choices[categorical1_centered.prior], + ) == expected_prior_min_upper + + assert ( + categorical2_centered.prior, + categorical2_centered.lower, + categorical2_centered.upper, + categorical2_centered.choices[categorical2_centered.prior], + ) == expected_prior_min_upper + + categorical1_centered.sample() + categorical2_centered.sample() diff --git a/tests/test_neps_space/test_neps_integration.py b/tests/test_neps_space/test_neps_integration.py new file mode 100644 index 000000000..8f18ace39 --- /dev/null +++ b/tests/test_neps_space/test_neps_integration.py @@ -0,0 +1,580 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from functools import partial + +import pytest + +import neps +from neps.optimizers import algorithms +from neps.space.neps_spaces.neps_space import ( + check_neps_space_compatibility, + convert_classic_to_neps_search_space, + convert_neps_to_classic_search_space, +) +from neps.space.neps_spaces.parameters import ( + Categorical, + ConfidenceLevel, + Float, + Integer, + IntegerFidelity, + Operation, + PipelineSpace, +) + + +def hyperparameter_pipeline_to_optimize( + float1: float, + float2: float, + categorical: int, + integer1: int, + integer2: int, +): + assert isinstance(float1, float) + assert isinstance(float2, float) + assert isinstance(categorical, int) + assert isinstance(integer1, int) + assert isinstance(integer2, int) + + objective_to_minimize = -float(float1 + float2 + categorical + integer1 + integer2) + assert isinstance(objective_to_minimize, float) + + return objective_to_minimize + + +class DemoHyperparameterSpace(PipelineSpace): + float1 = Float( + lower=0, + upper=1, + prior=0.1, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + float2 = Float( + lower=-10, + upper=10, + prior=0.1, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + categorical = Categorical( + choices=(0, 1), + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer1 = Integer( + lower=0, + upper=1, + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer2 = Integer( + lower=1, + upper=1000, + prior=10, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + + +class DemoHyperparameterWithFidelitySpace(PipelineSpace): + float1 = Float( + lower=0, + upper=1, + prior=0.1, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + float2 = Float( + lower=-10, + upper=10, + prior=0.1, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + categorical = Categorical( + choices=(0, 1), + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer1 = Integer( + lower=0, + upper=1, + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer2 = IntegerFidelity( + lower=1, + upper=1000, + ) + + +class DemoHyperparameterComplexSpace(PipelineSpace): + _small_float = Float( + lower=0, + upper=1, + prior=0.1, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + _big_float = Float( + lower=10, + upper=100, + prior=20, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + + float1 = Categorical( + choices=( + _small_float.resample(), + _big_float.resample(), + ), + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + float2 = Categorical( + choices=( + _small_float.resample(), + _big_float.resample(), + float1, + ), + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + categorical = Categorical( + choices=(0, 1), + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer1 = Integer( + lower=0, + upper=1, + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer2 = Integer( + lower=1, + upper=1000, + prior=10, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + + +@pytest.mark.parametrize( + "optimizer", + [ + partial(algorithms.neps_random_search, ignore_fidelity=True), + partial(algorithms.complex_random_search, ignore_fidelity=True), + ], +) +def test_hyperparameter_demo(optimizer): + pipeline_space = DemoHyperparameterSpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/hyperparameter_demo__{optimizer.func.__name__}" + + neps.run( + evaluate_pipeline=hyperparameter_pipeline_to_optimize, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + evaluations_to_spend=10, + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) + + +@pytest.mark.parametrize( + "optimizer", + [ + partial(algorithms.neps_random_search, ignore_fidelity=True), + partial(algorithms.complex_random_search, ignore_fidelity=True), + ], +) +def test_hyperparameter_with_fidelity_demo(optimizer): + pipeline_space = DemoHyperparameterWithFidelitySpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/hyperparameter_with_fidelity_demo__{optimizer.func.__name__}" + + neps.run( + evaluate_pipeline=hyperparameter_pipeline_to_optimize, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + evaluations_to_spend=10, + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) + + +@pytest.mark.parametrize( + "optimizer", + [ + partial(algorithms.neps_random_search, ignore_fidelity=True), + partial(algorithms.complex_random_search, ignore_fidelity=True), + ], +) +def test_hyperparameter_complex_demo(optimizer): + pipeline_space = DemoHyperparameterComplexSpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/hyperparameter_complex_demo__{optimizer.func.__name__}" + + neps.run( + evaluate_pipeline=hyperparameter_pipeline_to_optimize, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + overwrite_root_directory=True, + evaluations_to_spend=10, + ) + neps.status(root_directory, print_summary=True) + + +# ----------------------------------------- + + +class Model: + """A simple model that takes an inner function and a factor, + multiplies the result of the inner function by the factor. + """ + + def __init__( + self, + inner_function: Callable[[Sequence[float]], float], + factor: float, + ): + """Initialize the model with an inner function and a factor.""" + self.inner_function = inner_function + self.factor = factor + + def __call__(self, values: Sequence[float]) -> float: + return self.factor * self.inner_function(values) + + +class Sum: + """A simple inner function that sums the values.""" + + def __call__(self, values: Sequence[float]) -> float: + return sum(values) + + +class MultipliedSum: + """An inner function that sums the values and multiplies the result by a factor.""" + + def __init__(self, factor: float): + """Initialize the multiplied sum with a factor.""" + self.factor = factor + + def __call__(self, values: Sequence[float]) -> float: + return self.factor * sum(values) + + +def operation_pipeline_to_optimize(model: Model, some_hp: str): + assert isinstance(model, Model) + assert isinstance(model.factor, float) + assert isinstance(model.inner_function, Sum | MultipliedSum) + if isinstance(model.inner_function, MultipliedSum): + assert isinstance(model.inner_function.factor, float) + assert some_hp in {"hp1", "hp2"} + + values = list(range(1, 21)) + objective_to_minimize = model(values) + assert isinstance(objective_to_minimize, float) + + return objective_to_minimize + + +class DemoOperationSpace(PipelineSpace): + """A demonstration of how to use operations in a search space. + This space defines a model that can be optimized using different inner functions + and a factor. The model can be used to evaluate a set of values and return an objective to minimize. + """ + + # The way to sample `factor` values + _factor = Float( + lower=0, + upper=1, + prior=0.1, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + + # Sum + # Will be equivalent to something like + # `Sum()` + # Could have also been defined using the python `sum` function as + # `_sum = space.Operation(operator=lambda: sum)` + _sum = Operation(operator=Sum) + + # MultipliedSum + # Will be equivalent to something like + # `MultipliedSum(factor=0.2)` + _multiplied_sum = Operation( + operator=MultipliedSum, + kwargs={"factor": _factor.resample()}, + ) + + # Model + # Will be equivalent to something like one of + # `Model(Sum(), factor=0.1)` + # `Model(MultipliedSum(factor=0.2), factor=0.1)` + _inner_function = Categorical( + choices=(_sum, _multiplied_sum), + ) + model = Operation( + operator=Model, + args=(_inner_function,), + kwargs={"factor": _factor.resample()}, + ) + + # An additional hyperparameter + some_hp = Categorical( + choices=("hp1", "hp2"), + ) + + +@pytest.mark.parametrize( + "optimizer", + [ + algorithms.neps_random_search, + algorithms.complex_random_search, + ], +) +def test_operation_demo(optimizer): + pipeline_space = DemoOperationSpace() + root_directory = ( + f"tests_tmpdir/test_neps_spaces/results/operation_demo__{optimizer.__name__}" + ) + + neps.run( + evaluate_pipeline=operation_pipeline_to_optimize, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + evaluations_to_spend=10, + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) + + +# ===== Extended tests for newer NePS features ===== + + +# Test neps_hyperband with various PipelineSpaces +@pytest.mark.parametrize( + "optimizer", + [ + algorithms.neps_hyperband, + ], +) +def test_neps_hyperband_with_fidelity_demo(optimizer): + """Test neps_hyperband with a fidelity space.""" + pipeline_space = DemoHyperparameterWithFidelitySpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/neps_hyperband_fidelity_demo__{optimizer.__name__}" + + neps.run( + evaluate_pipeline=hyperparameter_pipeline_to_optimize, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + fidelities_to_spend=15, # Use fidelities_to_spend for multi-fidelity optimizers + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) + + +# Test PipelineSpace dynamic methods (add, remove, add_prior) +def test_pipeline_space_dynamic_methods(): + """Test PipelineSpace add, remove, and add_prior methods.""" + + # Create a basic space + class BasicSpace(PipelineSpace): + x = Float(lower=0.0, upper=1.0) + y = Integer(lower=1, upper=10) + + space = BasicSpace() + + # Test adding a new parameter + new_param = Categorical(choices=(True, False)) + space = space.add(new_param, "flag") + + # Verify the parameter was added + attrs = space.get_attrs() + assert "flag" in attrs + assert attrs["flag"] is new_param + + # Test adding a prior to an existing parameter + space = space.add_prior("x", prior=0.5, prior_confidence=ConfidenceLevel.HIGH) + + # Verify the prior was added + updated_attrs = space.get_attrs() + x_param = updated_attrs["x"] + assert x_param.has_prior + assert x_param.prior == 0.5 + assert x_param.prior_confidence == ConfidenceLevel.HIGH + + # Test removing a parameter + space = space.remove("y") + + # Verify the parameter was removed + final_attrs = space.get_attrs() + assert "y" not in final_attrs + assert "x" in final_attrs + assert "flag" in final_attrs + + +# Test space conversion functions +def test_space_conversion_functions(): + """Test conversion between classic and NePS spaces.""" + # Create a classic SearchSpace + classic_space = neps.SearchSpace( + { + "x": neps.HPOFloat(0.0, 1.0, prior=0.5, prior_confidence="medium"), + "y": neps.HPOInteger(1, 10, prior=5, prior_confidence="high"), + "z": neps.HPOCategorical(["a", "b", "c"], prior="b", prior_confidence="low"), + } + ) + + # Convert to NePS space + neps_space = convert_classic_to_neps_search_space(classic_space) + assert isinstance(neps_space, PipelineSpace) + + # Verify attributes are preserved + neps_attrs = neps_space.get_attrs() + assert len(neps_attrs) == 3 + assert all(name in neps_attrs for name in ["x", "y", "z"]) + + # Verify types and priors + assert isinstance(neps_attrs["x"], Float) + assert neps_attrs["x"].has_prior + assert neps_attrs["x"].prior == 0.5 + + assert isinstance(neps_attrs["y"], Integer) + assert neps_attrs["y"].has_prior + assert neps_attrs["y"].prior == 5 + + assert isinstance(neps_attrs["z"], Categorical) + assert neps_attrs["z"].has_prior + assert neps_attrs["z"].prior == 1 # Index of "b" in choices + + # Convert back to classic space + converted_back = convert_neps_to_classic_search_space(neps_space) + assert converted_back is not None + assert isinstance(converted_back, neps.SearchSpace) + + # Verify round-trip conversion preserves structure + classic_attrs = converted_back.elements + assert len(classic_attrs) == 3 + assert all(name in classic_attrs for name in ["x", "y", "z"]) + + +# Test algorithm compatibility checking +def test_algorithm_compatibility(): + """Test algorithm compatibility with different space types.""" + # Test NePS-only algorithms + neps_only_algorithms = [ + algorithms.neps_random_search, + algorithms.neps_hyperband, + algorithms.complex_random_search, + ] + + for algo in neps_only_algorithms: + compatibility = check_neps_space_compatibility(algo) + assert compatibility in [ + "neps", + "both", + ], f"Algorithm {algo.__name__} should be neps or both compatible" + + # Test classic algorithms that should work with both + both_compatible_algorithms = [ + algorithms.random_search, + algorithms.hyperband, + ] + + for algo in both_compatible_algorithms: + compatibility = check_neps_space_compatibility(algo) + assert compatibility in [ + "classic", + "both", + ], f"Algorithm {algo.__name__} should be classic or both compatible" + + +# Test with complex PipelineSpace containing Operations and Resample +def test_complex_neps_space_features(): + """Test complex NePS space features that cannot be converted to classic.""" + + class ComplexNepsSpace(PipelineSpace): + # Basic parameters + factor = Float( + lower=0.1, + upper=2.0, + prior=1.0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + + # Operation with resampled parameters + operation = Operation( + operator=lambda x, y: x * y, + args=(factor, factor.resample()), + ) + + # Categorical with operations as choices + choice = Categorical( + choices=(operation, factor), + prior=0, + prior_confidence=ConfidenceLevel.LOW, + ) + + space = ComplexNepsSpace() + + # This space should NOT be convertible to classic + converted = convert_neps_to_classic_search_space(space) + assert converted is None, "Complex NePS space should not be convertible to classic" + + # But should work with NePS-compatible algorithms + compatibility = check_neps_space_compatibility(algorithms.neps_random_search) + assert compatibility in ["neps", "both"] + + +# Test trajectory and metrics functionality +def test_trajectory_and_metrics(tmp_path): + """Test extended trajectory and best_config functionality.""" + + def evaluate_with_metrics(x: float, y: int) -> dict: + """Evaluation function that returns multiple metrics.""" + return { + "objective_to_minimize": x + y, + "accuracy": 1.0 - (x + y) / 11.0, # Dummy accuracy metric + "training_time": x * 10, # Dummy training time + "memory_usage": y * 100, # Dummy memory usage + } + + class MetricsSpace(PipelineSpace): + x = Float(lower=0.0, upper=1.0) + y = Integer(lower=1, upper=10) + + space = MetricsSpace() + root_directory = tmp_path / "metrics_test" + + # Run optimization + neps.run( + evaluate_pipeline=evaluate_with_metrics, + pipeline_space=space, + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=5, + overwrite_root_directory=True, + ) + + # Check that trajectory and best_config files exist and contain extended metrics + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + best_config_file = root_directory / "summary" / "best_config.txt" + + assert trajectory_file.exists(), "Trajectory file should exist" + assert best_config_file.exists(), "Best config file should exist" + + # Read and verify trajectory contains the standard format (not extended metrics in txt files) + trajectory_content = trajectory_file.read_text() + assert "Config ID:" in trajectory_content, "Trajectory should contain Config ID" + assert "Objective to minimize:" in trajectory_content, ( + "Trajectory should contain objective" + ) + assert "Cumulative evaluations:" in trajectory_content, ( + "Trajectory should contain cumulative evaluations" + ) + + # Read and verify best config contains the standard format + best_config_content = best_config_file.read_text() + assert "Config ID:" in best_config_content, "Best config should contain Config ID" + assert "Objective to minimize:" in best_config_content, ( + "Best config should contain objective" + ) diff --git a/tests/test_neps_space/test_neps_integration_priorband__max_cost.py b/tests/test_neps_space/test_neps_integration_priorband__max_cost.py new file mode 100644 index 000000000..77266e171 --- /dev/null +++ b/tests/test_neps_space/test_neps_integration_priorband__max_cost.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from functools import partial + +import numpy as np +import pytest + +import neps +from neps import algorithms +from neps.space.neps_spaces.parameters import ( + ConfidenceLevel, + Float, + Integer, + IntegerFidelity, + PipelineSpace, +) + +_COSTS = {} + + +def evaluate_pipeline(float1, float2, integer1, fidelity): + objective_to_minimize = -float(np.sum([float1, float2, integer1])) * fidelity + + key = (float1, float2, integer1) + old_cost = _COSTS.get(key, 0) + added_cost = fidelity - old_cost + + _COSTS[key] = fidelity + + return { + "objective_to_minimize": objective_to_minimize, + "cost": added_cost, + } + + +class DemoHyperparameterWithFidelitySpace(PipelineSpace): + float1 = Float( + lower=1, + upper=1000, + log=False, + prior=600, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + float2 = Float( + lower=-100, + upper=100, + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer1 = Integer( + lower=0, + upper=500, + prior=35, + prior_confidence=ConfidenceLevel.LOW, + ) + fidelity = IntegerFidelity( + lower=1, + upper=100, + ) + + +@pytest.mark.parametrize( + ("optimizer", "optimizer_name"), + [ + ( + partial(algorithms.neps_random_search, ignore_fidelity=True), + "neps_random_search", + ), + ( + partial(algorithms.complex_random_search, ignore_fidelity=True), + "neps_complex_random_search", + ), + ( + partial(algorithms.neps_priorband, base="successive_halving"), + "neps_priorband+successive_halving", + ), + ( + partial(algorithms.neps_priorband, base="asha"), + "neps_priorband+asha", + ), + ( + partial(algorithms.neps_priorband, base="async_hb"), + "neps_priorband+async_hb", + ), + ( + algorithms.neps_priorband, + "neps_priorband+hyperband", + ), + ], +) +def test_hyperparameter_with_fidelity_demo_new(optimizer, optimizer_name): + optimizer.__name__ = ( + "neps_priorband" if "priorband" in optimizer_name else optimizer_name + ) # Needed by NEPS later. + pipeline_space = DemoHyperparameterWithFidelitySpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/hyperparameter_with_fidelity__costs__{optimizer.__name__}" + + # Reset the _COSTS global, so they do not get mixed up between tests. + _COSTS.clear() + + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + cost_to_spend=100, + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) + + +@pytest.mark.parametrize( + ("optimizer", "optimizer_name"), + [ + ( + partial(algorithms.priorband, base="successive_halving"), + "old_priorband+successive_halving", + ), + ( + partial(algorithms.priorband, base="asha"), + "old_priorband+asha", + ), + ( + partial(algorithms.priorband, base="async_hb"), + "old_priorband+async_hb", + ), + ( + algorithms.priorband, + "old_priorband+hyperband", + ), + ], +) +def test_hyperparameter_with_fidelity_demo_old(optimizer, optimizer_name): + optimizer.__name__ = "priorband" # Needed by NEPS later. + pipeline_space = DemoHyperparameterWithFidelitySpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/hyperparameter_with_fidelity__costs__{optimizer.__name__}" + + # Reset the _COSTS global, so they do not get mixed up between tests. + _COSTS.clear() + + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + cost_to_spend=100, + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) diff --git a/tests/test_neps_space/test_neps_integration_priorband__max_evals.py b/tests/test_neps_space/test_neps_integration_priorband__max_evals.py new file mode 100644 index 000000000..828c82e37 --- /dev/null +++ b/tests/test_neps_space/test_neps_integration_priorband__max_evals.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from functools import partial + +import numpy as np +import pytest + +import neps +from neps.optimizers import algorithms +from neps.space.neps_spaces.parameters import ( + ConfidenceLevel, + Float, + Integer, + IntegerFidelity, + PipelineSpace, +) + + +def evaluate_pipeline(float1, float2, integer1, fidelity): + return -float(np.sum([float1, float2, integer1])) * fidelity + + +class DemoHyperparameterWithFidelitySpace(PipelineSpace): + float1 = Float( + lower=1, + upper=1000, + log=False, + prior=600, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + float2 = Float( + lower=-100, + upper=100, + prior=0, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + integer1 = Integer( + lower=0, + upper=500, + prior=35, + prior_confidence=ConfidenceLevel.LOW, + ) + fidelity = IntegerFidelity( + lower=1, + upper=100, + ) + + +@pytest.mark.parametrize( + ("optimizer", "optimizer_name"), + [ + ( + partial(algorithms.neps_random_search, ignore_fidelity=True), + "neps_random_search", + ), + ( + partial(algorithms.complex_random_search, ignore_fidelity=True), + "neps_complex_random_search", + ), + ( + partial(algorithms.neps_priorband, base="successive_halving"), + "neps_priorband+successive_halving", + ), + ( + partial(algorithms.neps_priorband, base="asha"), + "neps_priorband+asha", + ), + ( + partial(algorithms.neps_priorband, base="async_hb"), + "neps_priorband+async_hb", + ), + ( + algorithms.neps_priorband, + "neps_priorband+hyperband", + ), + ], +) +def test_hyperparameter_with_fidelity_demo_new(optimizer, optimizer_name): + optimizer.__name__ = ( + "neps_priorband" if "priorband" in optimizer_name else optimizer_name + ) # Needed by NEPS later. + pipeline_space = DemoHyperparameterWithFidelitySpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/hyperparameter_with_fidelity__evals__{optimizer.__name__}" + + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + fidelities_to_spend=50 if "priorband" in optimizer.__name__ else None, + evaluations_to_spend=50 if "priorband" not in optimizer.__name__ else None, + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) + + +@pytest.mark.parametrize( + ("optimizer", "optimizer_name"), + [ + ( + partial(algorithms.priorband, base="successive_halving"), + "old_priorband+successive_halving", + ), + ( + partial(algorithms.priorband, base="asha"), + "old_priorband+asha", + ), + ( + partial(algorithms.priorband, base="async_hb"), + "old_priorband+async_hb", + ), + ( + algorithms.priorband, + "old_priorband+hyperband", + ), + ], +) +def test_hyperparameter_with_fidelity_demo_old(optimizer, optimizer_name): + optimizer.__name__ = "priorband" # Needed by NEPS later. + pipeline_space = DemoHyperparameterWithFidelitySpace() + root_directory = f"tests_tmpdir/test_neps_spaces/results/hyperparameter_with_fidelity__evals__{optimizer.__name__}" + + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=pipeline_space, + optimizer=optimizer, + root_directory=root_directory, + fidelities_to_spend=50, + overwrite_root_directory=True, + ) + neps.status(root_directory, print_summary=True) diff --git a/tests/test_neps_space/test_pipeline_space_methods.py b/tests/test_neps_space/test_pipeline_space_methods.py new file mode 100644 index 000000000..af90b2b52 --- /dev/null +++ b/tests/test_neps_space/test_pipeline_space_methods.py @@ -0,0 +1,394 @@ +"""Tests for PipelineSpace dynamic methods (add, remove, add_prior).""" + +from __future__ import annotations + +import pytest + +from neps.space.neps_spaces.parameters import ( + Categorical, + ConfidenceLevel, + Float, + Integer, + IntegerFidelity, + Operation, + PipelineSpace, + Resample, +) + + +class BasicSpace(PipelineSpace): + """Basic space for testing dynamic methods.""" + + x = Float(lower=0.0, upper=1.0) + y = Integer(lower=1, upper=10) + z = Categorical(choices=("a", "b", "c")) + + +class SpaceWithPriors(PipelineSpace): + """Space with existing priors for testing.""" + + x = Float(lower=0.0, upper=1.0, prior=0.5, prior_confidence=ConfidenceLevel.MEDIUM) + y = Integer(lower=1, upper=10, prior=5, prior_confidence=ConfidenceLevel.HIGH) + z = Categorical( + choices=("a", "b", "c"), prior=1, prior_confidence=ConfidenceLevel.LOW + ) + + +# ===== Test add method ===== + + +def test_add_method_basic(): + """Test basic functionality of the add method.""" + space = BasicSpace() + original_attrs = space.get_attrs() + + # Add a new parameter + new_param = Float(lower=10.0, upper=20.0) + updated_space = space.add(new_param, "new_float") + + # Original space should be unchanged + assert space.get_attrs() == original_attrs + + # Updated space should have the new parameter + updated_attrs = updated_space.get_attrs() + assert "new_float" in updated_attrs + assert updated_attrs["new_float"] is new_param + assert len(updated_attrs) == len(original_attrs) + 1 + + +def test_add_method_different_types(): + """Test adding different parameter types.""" + space = BasicSpace() + + # Add Integer + space = space.add(Integer(lower=0, upper=100), "new_int") + assert "new_int" in space.get_attrs() + assert isinstance(space.get_attrs()["new_int"], Integer) + + # Add Categorical + space = space.add(Categorical(choices=(True, False)), "new_cat") + assert "new_cat" in space.get_attrs() + assert isinstance(space.get_attrs()["new_cat"], Categorical) + + # Add Operation + op = Operation(operator=lambda x: x * 2, args=(space.get_attrs()["x"],)) + space = space.add(op, "new_op") + assert "new_op" in space.get_attrs() + assert isinstance(space.get_attrs()["new_op"], Operation) + + # Add Resample + resampled = space.get_attrs()["x"].resample() + space = space.add(resampled, "new_resampled") + assert "new_resampled" in space.get_attrs() + assert isinstance(space.get_attrs()["new_resampled"], Resample) + + +def test_add_method_with_default_name(): + """Test add method with automatic name generation.""" + space = BasicSpace() + original_count = len(space.get_attrs()) + + # Add without specifying name + new_param = Float(lower=5.0, upper=15.0) + updated_space = space.add(new_param) + + updated_attrs = updated_space.get_attrs() + assert len(updated_attrs) == original_count + 1 + + # Should have generated a name like "param_4" + generated_names = [name for name in updated_attrs if name.startswith("param_")] + assert len(generated_names) >= 1 + + +def test_add_method_duplicate_parameter(): + """Test adding a parameter with an existing name but same content.""" + space = BasicSpace() + + # Add the same parameter that already exists + existing_param = space.get_attrs()["x"] + updated_space = space.add(existing_param, "x") + + # Should work without error + assert updated_space.get_attrs()["x"] is existing_param + + +def test_add_method_conflicting_parameter(): + """Test adding a different parameter with an existing name.""" + space = BasicSpace() + + # Try to add a different parameter with existing name + different_param = Integer(lower=0, upper=5) # Different from existing "x" + + with pytest.raises(ValueError, match="A different parameter with the name"): + space.add(different_param, "x") + + +def test_add_method_chaining(): + """Test chaining multiple add operations.""" + space = BasicSpace() + + # Chain multiple additions + final_space = ( + space.add(Float(lower=100.0, upper=200.0), "param1") + .add(Integer(lower=0, upper=50), "param2") + .add(Categorical(choices=(1, 2, 3)), "param3") + ) + + attrs = final_space.get_attrs() + assert "param1" in attrs + assert "param2" in attrs + assert "param3" in attrs + assert len(attrs) == 6 # 3 original + 3 new + + +# ===== Test remove method ===== + + +def test_remove_method_basic(): + """Test basic functionality of the remove method.""" + space = BasicSpace() + original_attrs = space.get_attrs() + + # Remove a parameter + updated_space = space.remove("y") + + # Original space should be unchanged + assert space.get_attrs() == original_attrs + + # Updated space should not have the removed parameter + updated_attrs = updated_space.get_attrs() + assert "y" not in updated_attrs + assert "x" in updated_attrs + assert "z" in updated_attrs + assert len(updated_attrs) == len(original_attrs) - 1 + + +def test_remove_method_nonexistent_parameter(): + """Test removing a parameter that doesn't exist.""" + space = BasicSpace() + + with pytest.raises(ValueError, match="No parameter with the name"): + space.remove("nonexistent") + + +def test_remove_method_chaining(): + """Test chaining multiple remove operations.""" + space = BasicSpace() + + # Chain multiple removals + final_space = space.remove("x").remove("y") + + attrs = final_space.get_attrs() + assert "x" not in attrs + assert "y" not in attrs + assert "z" in attrs + assert len(attrs) == 1 + + +def test_remove_all_parameters(): + """Test removing all parameters from a space.""" + space = BasicSpace() + + # Remove all parameters + empty_space = space.remove("x").remove("y").remove("z") + + attrs = empty_space.get_attrs() + assert len(attrs) == 0 + + +# ===== Test add_prior method ===== + + +def test_add_prior_method_basic(): + """Test basic functionality of the add_prior method.""" + space = BasicSpace() + space.get_attrs() + + # Add prior to a parameter without prior + updated_space = space.add_prior("x", prior=0.5, prior_confidence=ConfidenceLevel.HIGH) + + # Original space should be unchanged + original_x = space.get_attrs()["x"] + assert not original_x.has_prior + + # Updated space should have the prior + updated_x = updated_space.get_attrs()["x"] + assert updated_x.has_prior + assert updated_x.prior == 0.5 + assert updated_x.prior_confidence == ConfidenceLevel.HIGH + + +def test_add_prior_method_different_types(): + """Test adding priors to different parameter types.""" + space = BasicSpace() + + # Add prior to Float + space = space.add_prior("x", prior=0.75, prior_confidence=ConfidenceLevel.MEDIUM) + x_param = space.get_attrs()["x"] + assert x_param.has_prior + assert x_param.prior == 0.75 + + # Add prior to Integer + space = space.add_prior("y", prior=7, prior_confidence=ConfidenceLevel.HIGH) + y_param = space.get_attrs()["y"] + assert y_param.has_prior + assert y_param.prior == 7 + + # Add prior to Categorical + space = space.add_prior("z", prior=2, prior_confidence=ConfidenceLevel.LOW) + z_param = space.get_attrs()["z"] + assert z_param.has_prior + assert z_param.prior == 2 + + +def test_add_prior_method_string_confidence(): + """Test add_prior with string confidence levels.""" + space = BasicSpace() + + # Test with string confidence levels + space = space.add_prior("x", prior=0.3, prior_confidence="low") + x_param = space.get_attrs()["x"] + assert x_param.has_prior + assert x_param.prior == 0.3 + assert x_param.prior_confidence == ConfidenceLevel.LOW + + space = space.add_prior("y", prior=8, prior_confidence="medium") + y_param = space.get_attrs()["y"] + assert y_param.prior_confidence == ConfidenceLevel.MEDIUM + + space = space.add_prior("z", prior=0, prior_confidence="high") + z_param = space.get_attrs()["z"] + assert z_param.prior_confidence == ConfidenceLevel.HIGH + + +def test_add_prior_method_nonexistent_parameter(): + """Test adding prior to a parameter that doesn't exist.""" + space = BasicSpace() + + with pytest.raises(ValueError, match="No parameter with the name"): + space.add_prior("nonexistent", prior=0.5, prior_confidence=ConfidenceLevel.MEDIUM) + + +def test_add_prior_method_already_has_prior(): + """Test adding prior to a parameter that already has one.""" + space = SpaceWithPriors() + + with pytest.raises(ValueError, match="already has a prior"): + space.add_prior("x", prior=0.8, prior_confidence=ConfidenceLevel.LOW) + + +def test_add_prior_method_unsupported_type(): + """Test adding prior to unsupported parameter types.""" + # Create space with an Operation (which doesn't support priors) + space = BasicSpace() + op = Operation(operator=lambda x: x * 2, args=(space.get_attrs()["x"],)) + space = space.add(op, "operation_param") + + with pytest.raises(ValueError, match="does not support priors"): + space.add_prior( + "operation_param", prior=0.5, prior_confidence=ConfidenceLevel.MEDIUM + ) + + +# ===== Test combined operations ===== + + +def test_combined_operations(): + """Test combining add, remove, and add_prior operations.""" + space = BasicSpace() + + # Complex chain of operations + final_space = ( + space.add(Float(lower=50.0, upper=100.0), "new_param") + .remove("y") + .add_prior("x", prior=0.25, prior_confidence=ConfidenceLevel.HIGH) + .add_prior("new_param", prior=75.0, prior_confidence=ConfidenceLevel.MEDIUM) + .add(Integer(lower=0, upper=10), "another_param") + ) + + attrs = final_space.get_attrs() + + # Check structure + assert "x" in attrs + assert "y" not in attrs # Removed + assert "z" in attrs + assert "new_param" in attrs + assert "another_param" in attrs + + # Check priors + assert attrs["x"].has_prior + assert attrs["x"].prior == 0.25 + assert attrs["new_param"].has_prior + assert attrs["new_param"].prior == 75.0 + assert not attrs["z"].has_prior + assert not attrs["another_param"].has_prior + + +def test_immutability(): + """Test that all operations return new instances and don't modify originals.""" + original_space = BasicSpace() + original_attrs = original_space.get_attrs() + + # Perform various operations + space1 = original_space.add(Float(lower=0.0, upper=1.0), "temp") + space2 = original_space.remove("x") + space3 = original_space.add_prior("y", prior=5, prior_confidence=ConfidenceLevel.HIGH) + + # Original should be unchanged + assert original_space.get_attrs() == original_attrs + assert not original_space.get_attrs()["y"].has_prior + + # Each operation should create different instances + assert space1 is not original_space + assert space2 is not original_space + assert space3 is not original_space + assert space1 is not space2 + assert space2 is not space3 + + +def test_fidelity_operations(): + """Test operations with fidelity parameters.""" + + class FidelitySpace(PipelineSpace): + x = Float(lower=0.0, upper=1.0) + epochs = IntegerFidelity(lower=1, upper=100) + + space = FidelitySpace() + + # Add another parameter (non-fidelity since add doesn't support Fidelity directly) + new_param = Integer(lower=1, upper=50) + space = space.add(new_param, "batch_size") + + # Check that original fidelity is preserved + fidelity_attrs = space.fidelity_attrs + assert "epochs" in fidelity_attrs + assert len(fidelity_attrs) == 1 + + # Remove the fidelity parameter + space = space.remove("epochs") + fidelity_attrs = space.fidelity_attrs + assert "epochs" not in fidelity_attrs + assert len(fidelity_attrs) == 0 + + # Regular parameters should still be there + regular_attrs = space.get_attrs() + assert "x" in regular_attrs + assert "batch_size" in regular_attrs + + +def test_space_string_representation(): + """Test that string representation works after operations.""" + space = BasicSpace() + + # Perform operations + modified_space = ( + space.add(Float(lower=10.0, upper=20.0), "added_param") + .remove("y") + .add_prior("x", prior=0.8, prior_confidence=ConfidenceLevel.LOW) + ) + + # Should be able to get string representation without error + str_repr = str(modified_space) + assert "BasicSpace" in str_repr + assert "added_param = " in str_repr + assert "y = " not in str_repr # Should be removed diff --git a/tests/test_neps_space/test_search_space__fidelity.py b/tests/test_neps_space/test_search_space__fidelity.py new file mode 100644 index 000000000..922999785 --- /dev/null +++ b/tests/test_neps_space/test_search_space__fidelity.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import re + +import pytest + +import neps.space.neps_spaces.sampling +from neps.space.neps_spaces import neps_space +from neps.space.neps_spaces.parameters import ( + ConfidenceLevel, + Fidelity, + Float, + Integer, + IntegerFidelity, + PipelineSpace, +) + + +class DemoHyperparametersWithFidelitySpace(PipelineSpace): + constant1: int = 42 + float1 = Float( + lower=0, + upper=1, + prior=0.1, + prior_confidence=ConfidenceLevel.MEDIUM, + ) + fidelity_integer1 = IntegerFidelity( + lower=1, + upper=1000, + ) + + +def test_fidelity_creation_raises_when_domain_has_prior(): + # Creating a fidelity object with a domain that has a prior should not be possible. + with pytest.raises( + ValueError, + match=re.escape("The domain of a Fidelity can not have priors, has prior: 10"), + ): + Fidelity( + domain=Integer( + lower=1, + upper=1000, + prior=10, + prior_confidence=ConfidenceLevel.MEDIUM, + ), + ) + + +def test_fidelity_resolution_raises_when_resolved_with_no_environment_value(): + pipeline = DemoHyperparametersWithFidelitySpace() + + # Resolve a pipeline which contains a fidelity with an empty environment. + with pytest.raises( + ValueError, + match=re.escape( + "No value is available in the environment for fidelity 'fidelity_integer1'.", + ), + ): + neps_space.resolve(pipeline=pipeline) + + +def test_fidelity_resolution_raises_when_resolved_with_invalid_value(): + pipeline = DemoHyperparametersWithFidelitySpace() + + # Resolve a pipeline which contains a fidelity, + # with an environment value for it, that is out of the allowed range. + with pytest.raises( + ValueError, + match=re.escape( + "Value for fidelity with name 'fidelity_integer1' is outside its allowed" + " range [1, 1000]. Received: -10." + ), + ): + neps_space.resolve( + pipeline=pipeline, + environment_values={"fidelity_integer1": -10}, + ) + + +def test_fidelity_resolution_works(): + pipeline = DemoHyperparametersWithFidelitySpace() + + # Resolve a pipeline which contains a fidelity, + # with a valid value for it in the environment. + resolved_pipeline, _ = neps_space.resolve( + pipeline=pipeline, + environment_values={"fidelity_integer1": 10}, + ) + + assert resolved_pipeline.constant1 == 42 + assert ( + 0.0 <= float(str(resolved_pipeline.float1)) <= 1.0 + ) # 0.0 <= resolved_pipeline.float1 <= 1.0 also works, but gives a type warning + assert resolved_pipeline.fidelity_integer1 == 10 + + +def test_fidelity_resolution_with_context_works(): + pipeline = DemoHyperparametersWithFidelitySpace() + + samplings_to_make = { + "Resolvable.float1::float__0_1_False": 0.5, + } + environment_values = { + "fidelity_integer1": 10, + } + + # Resolve a pipeline which contains a fidelity, + # with a valid value for it in the environment. + resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=pipeline, + domain_sampler=neps.space.neps_spaces.sampling.OnlyPredefinedValuesSampler( + predefined_samplings=samplings_to_make, + ), + environment_values=environment_values, + ) + + assert resolved_pipeline.constant1 == 42 + assert resolved_pipeline.float1 == 0.5 + assert resolved_pipeline.fidelity_integer1 == 10 + + assert resolution_context.samplings_made == samplings_to_make + assert resolution_context.environment_values == environment_values diff --git a/tests/test_neps_space/test_search_space__grammar_like.py b/tests/test_neps_space/test_search_space__grammar_like.py new file mode 100644 index 000000000..3b43e2422 --- /dev/null +++ b/tests/test_neps_space/test_search_space__grammar_like.py @@ -0,0 +1,512 @@ +from __future__ import annotations + +import pytest + +import neps.space.neps_spaces.sampling +from neps.space.neps_spaces import neps_space, string_formatter +from neps.space.neps_spaces.parameters import ( + ByName, + Categorical, + Operation, + PipelineSpace, +) + + +class GrammarLike(PipelineSpace): + _id = Operation(operator="Identity") + _three = Operation(operator="Conv2D-3") + _one = Operation(operator="Conv2D-1") + _reluconvbn = Operation(operator="ReLUConvBN") + + _O = Categorical(choices=(_three, _one, _id)) + + _C0 = Operation( + operator="Sequential", + args=(_O.resample(),), + ) + _C1 = Operation( + operator="Sequential", + args=( + _O.resample(), + ByName("S").resample(), + _reluconvbn, + ), + ) + _C2 = Operation( + operator="Sequential", + args=( + _O.resample(), + ByName("S").resample(), + ), + ) + _C3 = Operation( + operator="Sequential", + args=(ByName("S").resample(),), + ) + _C = Categorical( + choices=( + _C0.resample(), + _C1.resample(), + _C2.resample(), + _C3.resample(), + ), + ) + + _S0 = Operation( + operator="Sequential", + args=(_C.resample(),), + ) + _S1 = Operation( + operator="Sequential", + args=(_reluconvbn,), + ) + _S2 = Operation( + operator="Sequential", + args=(ByName("S").resample(),), + ) + _S3 = Operation( + operator="Sequential", + args=( + ByName("S").resample(), + _C.resample(), + ), + ) + _S4 = Operation( + operator="Sequential", + args=( + _O.resample(), + _O.resample(), + _O.resample(), + ), + ) + _S5 = Operation( + operator="Sequential", + args=( + ByName("S").resample(), + ByName("S").resample(), + _O.resample(), + _O.resample(), + _O.resample(), + _O.resample(), + _O.resample(), + _O.resample(), + ), + ) + S = Categorical( + choices=( + _S0.resample(), + _S1.resample(), + _S2.resample(), + _S3.resample(), + _S4.resample(), + _S5.resample(), + ), + ) + + +class GrammarLikeAlt(PipelineSpace): + _id = Operation(operator="Identity") + _three = Operation(operator="Conv2D-3") + _one = Operation(operator="Conv2D-1") + _reluconvbn = Operation(operator="ReLUConvBN") + + _O = Categorical(choices=(_three, _one, _id)) + + _C_ARGS = Categorical( + choices=( + (_O.resample(),), + ( + _O.resample(), + ByName("S").resample(), + _reluconvbn, + ), + ( + _O.resample(), + ByName("S").resample(), + ), + (ByName("S").resample(),), + ), + ) + _C = Operation( + operator="Sequential", + args=_C_ARGS.resample(), + ) + + _S_ARGS = Categorical( + choices=( + (_C.resample(),), + (_reluconvbn,), + (ByName("S").resample(),), + ( + ByName("S").resample(), + _C.resample(), + ), + ( + _O.resample(), + _O.resample(), + _O.resample(), + ), + ( + ByName("S").resample(), + ByName("S").resample(), + _O.resample(), + _O.resample(), + _O.resample(), + _O.resample(), + _O.resample(), + _O.resample(), + ), + ), + ) + S = Operation( + operator="Sequential", + args=_S_ARGS.resample(), + ) + + +@pytest.mark.repeat(500) +def test_resolve(): + pipeline = GrammarLike() + + try: + resolved_pipeline, _ = neps_space.resolve(pipeline) + except RecursionError: + pytest.xfail("XFAIL due to too much recursion.") + + s = resolved_pipeline.S + s_config_string = string_formatter.format_value(s) + assert s_config_string + + +@pytest.mark.repeat(500) +def test_resolve_alt(): + pipeline = GrammarLikeAlt() + + try: + resolved_pipeline, _ = neps_space.resolve(pipeline) + except RecursionError: + pytest.xfail("XFAIL due to too much recursion.") + + s = resolved_pipeline.S + s_config_string = string_formatter.format_value(s) + assert s_config_string + + +def test_resolve_context(): + samplings_to_make = { + "Resolvable.S::categorical__6": 5, + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__6": ( + 3 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical::categorical__4": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical::categorical__6": ( + 5 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__6": ( + 0 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__4": ( + 3 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__6": ( + 4 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[2].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[2].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[3].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[4].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[5].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[6].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[7].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[1].resampled_categorical::categorical__6": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__6": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__6": ( + 0 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__4": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[1].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[1].resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[2].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[3].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[4].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[5].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[6].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.sampled_value.resampled_operation.args.sequence[7].resampled_categorical::categorical__3": ( + 1 + ), + } + + pipeline = GrammarLike() + + resolved_pipeline, resolution_context = neps_space.resolve( + pipeline, + domain_sampler=neps.space.neps_spaces.sampling.OnlyPredefinedValuesSampler( + predefined_samplings=samplings_to_make, + ), + ) + sampled_values = resolution_context.samplings_made + + assert resolved_pipeline is not None + assert sampled_values is not None + assert sampled_values is not samplings_to_make + assert sampled_values == samplings_to_make + assert list(sampled_values.items()) == list(samplings_to_make.items()) + + # we should have made exactly those samplings + assert sampled_values == samplings_to_make + + s = resolved_pipeline.S + s_config_string = string_formatter.format_value(s) + assert s_config_string + # Verify the config contains expected operation names (format may be compact or multiline) + assert "Sequential" in s_config_string + assert "ReLUConvBN" in s_config_string + assert "Conv2D-3" in s_config_string + assert "Identity" in s_config_string + assert "Conv2D-1" in s_config_string + + +def test_resolve_context_alt(): + samplings_to_make = { + "Resolvable.S.args.resampled_categorical::categorical__6": 3, + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 5 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__4": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[2].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[3].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[4].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[5].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[6].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[7].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__4": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__6": ( + 5 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 5 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 3 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__4": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__6": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[2].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[3].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[4].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[5].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[6].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[7].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__6": ( + 5 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__6": ( + 3 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__6": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical::categorical__4": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical::categorical__4": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[0].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[2].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[3].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[4].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[5].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[6].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[7].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[2].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[3].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[4].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[5].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[6].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.S.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[1].resampled_operation.args.resampled_categorical.sampled_value.sequence[7].resampled_categorical::categorical__3": ( + 0 + ), + } + + pipeline = GrammarLikeAlt() + + resolved_pipeline, resolution_context = neps_space.resolve( + pipeline, + domain_sampler=neps.space.neps_spaces.sampling.OnlyPredefinedValuesSampler( + predefined_samplings=samplings_to_make, + ), + ) + sampled_values = resolution_context.samplings_made + + assert resolved_pipeline is not None + assert sampled_values is not None + assert sampled_values is not samplings_to_make + assert sampled_values == samplings_to_make + assert list(sampled_values.items()) == list(samplings_to_make.items()) + + # we should have made exactly those samplings + assert sampled_values == samplings_to_make + + s = resolved_pipeline.S + s_config_string = string_formatter.format_value(s) + assert s_config_string + # Verify the config contains expected operation names (format may be compact or multiline) + assert "Sequential" in s_config_string + assert "ReLUConvBN" in s_config_string + assert "Conv2D-1" in s_config_string + assert "Identity" in s_config_string + + +def test_string_resample_raises_error(): + """Test that using Resample with a string raises a TypeError.""" + from neps.space.neps_spaces.parameters import Resample + + with pytest.raises(TypeError, match="Resample does not accept plain strings"): + Resample("test_param") diff --git a/tests/test_neps_space/test_search_space__hnas_like.py b/tests/test_neps_space/test_search_space__hnas_like.py new file mode 100644 index 000000000..c4d867fb1 --- /dev/null +++ b/tests/test_neps_space/test_search_space__hnas_like.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import pytest + +import neps.space.neps_spaces.sampling +from neps.space.neps_spaces import neps_space, string_formatter +from neps.space.neps_spaces.parameters import ( + Categorical, + Float, + Operation, + PipelineSpace, +) + + +class HNASLikePipeline(PipelineSpace): + """Based on the `hierarchical+shared` variant (cell block is shared everywhere). + Across _CONVBLOCK items, _ACT and _CONV also shared. Only the _NORM changes. + + Additionally, this variant now has a PReLU operation with a float hyperparameter (init). + The same value of that hyperparameter would is used everywhere a _PRELU is used. + """ + + # ------------------------------------------------------ + # Adding `PReLU` with a float hyperparameter `init` + # Note that the sampled `_prelu_init_value` will be shared across all `_PRELU` uses, + # since no `Resample` was requested for it + _prelu_init_value = Float(lower=0.1, upper=0.9) + _PRELU = Operation( + operator="ACT prelu", + kwargs={"init": _prelu_init_value}, + ) + # ------------------------------------------------------ + + # Added `_PRELU` to the possible `_ACT` choices + _ACT = Categorical( + choices=( + Operation(operator="ACT relu"), + Operation(operator="ACT hardswish"), + Operation(operator="ACT mish"), + _PRELU, + ), + ) + _CONV = Categorical( + choices=( + Operation(operator="CONV conv1x1"), + Operation(operator="CONV conv3x3"), + Operation(operator="CONV dconv3x3"), + ), + ) + _NORM = Categorical( + choices=( + Operation(operator="NORM batch"), + Operation(operator="NORM instance"), + Operation(operator="NORM layer"), + ), + ) + + _CONVBLOCK = Operation( + operator="CONVBLOCK Sequential3", + args=( + _ACT, + _CONV, + _NORM.resample(), + ), + ) + _CONVBLOCK_FULL = Operation( + operator="OPS Sequential1", + args=(_CONVBLOCK.resample(),), + ) + _OP = Categorical( + choices=( + Operation(operator="OPS zero"), + Operation(operator="OPS id"), + Operation(operator="OPS avg_pool"), + _CONVBLOCK_FULL.resample(), + ), + ) + + CL = Operation( + operator="CELL Cell", + args=( + _OP.resample(), + _OP.resample(), + _OP.resample(), + _OP.resample(), + _OP.resample(), + _OP.resample(), + ), + ) + + _C = Categorical( + choices=( + Operation(operator="C Sequential2", args=(CL, CL)), + Operation(operator="C Sequential3", args=(CL, CL, CL)), + Operation(operator="C Residual2", args=(CL, CL, CL)), + ), + ) + + _RESBLOCK = Operation(operator="resBlock") + _DOWN = Categorical( + choices=( + Operation(operator="DOWN Sequential2", args=(CL, _RESBLOCK)), + Operation(operator="DOWN Sequential3", args=(CL, CL, _RESBLOCK)), + Operation(operator="DOWN Residual2", args=(CL, _RESBLOCK, _RESBLOCK)), + ), + ) + + _D0 = Categorical( + choices=( + Operation( + operator="D0 Sequential3", + args=( + _C.resample(), + _C.resample(), + CL, + ), + ), + Operation( + operator="D0 Sequential4", + args=( + _C.resample(), + _C.resample(), + _C.resample(), + CL, + ), + ), + Operation( + operator="D0 Residual3", + args=( + _C.resample(), + _C.resample(), + CL, + CL, + ), + ), + ), + ) + _D1 = Categorical( + choices=( + Operation( + operator="D1 Sequential3", + args=( + _C.resample(), + _C.resample(), + _DOWN.resample(), + ), + ), + Operation( + operator="D1 Sequential4", + args=( + _C.resample(), + _C.resample(), + _C.resample(), + _DOWN.resample(), + ), + ), + Operation( + operator="D1 Residual3", + args=( + _C.resample(), + _C.resample(), + _DOWN.resample(), + _DOWN.resample(), + ), + ), + ), + ) + + _D2 = Categorical( + choices=( + Operation( + operator="D2 Sequential3", + args=( + _D1.resample(), + _D1.resample(), + _D0.resample(), + ), + ), + Operation( + operator="D2 Sequential3", + args=( + _D0.resample(), + _D1.resample(), + _D1.resample(), + ), + ), + Operation( + operator="D2 Sequential4", + args=( + _D1.resample(), + _D1.resample(), + _D0.resample(), + _D0.resample(), + ), + ), + ), + ) + + ARCH: Operation = _D2 + + +@pytest.mark.repeat(500) +def test_hnas_like(): + pipeline = HNASLikePipeline() + + resolved_pipeline, resolution_context = neps_space.resolve(pipeline) + assert resolved_pipeline is not None + assert resolution_context.samplings_made is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ("CL", "ARCH") + + +@pytest.mark.repeat(500) +def test_hnas_like_string(): + pipeline = HNASLikePipeline() + + resolved_pipeline, _ = neps_space.resolve(pipeline) + + arch = resolved_pipeline.ARCH + arch_config_string = string_formatter.format_value(arch) + assert arch_config_string + + cl = resolved_pipeline.CL + cl_config_string = string_formatter.format_value(cl) + assert cl_config_string + + +def test_hnas_like_context(): + samplings_to_make = { + "Resolvable.CL.args.sequence[0].resampled_categorical::categorical__4": 3, + "Resolvable.CL.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_operation.args.sequence[0]::categorical__4": ( + 0 + ), + "Resolvable.CL.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_operation.args.sequence[1]::categorical__3": ( + 2 + ), + "Resolvable.CL.args.sequence[0].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_operation.args.sequence[2].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.CL.args.sequence[1].resampled_categorical::categorical__4": 0, + "Resolvable.CL.args.sequence[2].resampled_categorical::categorical__4": 1, + "Resolvable.CL.args.sequence[3].resampled_categorical::categorical__4": 2, + "Resolvable.CL.args.sequence[4].resampled_categorical::categorical__4": 3, + "Resolvable.CL.args.sequence[4].resampled_categorical.sampled_value.resampled_operation.args.sequence[0].resampled_operation.args.sequence[2].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.CL.args.sequence[5].resampled_categorical::categorical__4": 0, + "Resolvable.ARCH::categorical__3": 1, + "Resolvable.ARCH.sampled_value.args.sequence[0].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.ARCH.sampled_value.args.sequence[0].resampled_categorical.sampled_value.args.sequence[0].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.ARCH.sampled_value.args.sequence[0].resampled_categorical.sampled_value.args.sequence[1].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.ARCH.sampled_value.args.sequence[1].resampled_categorical::categorical__3": ( + 2 + ), + "Resolvable.ARCH.sampled_value.args.sequence[1].resampled_categorical.sampled_value.args.sequence[0].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.ARCH.sampled_value.args.sequence[1].resampled_categorical.sampled_value.args.sequence[1].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.ARCH.sampled_value.args.sequence[1].resampled_categorical.sampled_value.args.sequence[2].resampled_categorical::categorical__3": ( + 0 + ), + "Resolvable.ARCH.sampled_value.args.sequence[1].resampled_categorical.sampled_value.args.sequence[3].resampled_categorical::categorical__3": ( + 1 + ), + "Resolvable.ARCH.sampled_value.args.sequence[2].resampled_categorical::categorical__3": ( + 2 + ), + } + + pipeline = HNASLikePipeline() + + resolved_pipeline, resolution_context = neps_space.resolve( + pipeline=pipeline, + domain_sampler=neps.space.neps_spaces.sampling.OnlyPredefinedValuesSampler( + predefined_samplings=samplings_to_make, + ), + ) + sampled_values = resolution_context.samplings_made + + assert resolved_pipeline is not None + assert sampled_values is not None + assert sampled_values is not samplings_to_make + assert sampled_values == samplings_to_make + assert list(sampled_values.items()) == list(samplings_to_make.items()) + + # we should have made exactly those samplings + assert sampled_values == samplings_to_make + + cl = resolved_pipeline.CL + cl_config_string = string_formatter.format_value(cl) + assert cl_config_string + # The new formatter outputs operations in full rather than using sharing references + # Check for essential elements instead of exact format + assert "Cell(" in cl_config_string + assert "Sequential" in cl_config_string + assert "relu" in cl_config_string + assert "dconv3x3" in cl_config_string + assert "NORM batch" in cl_config_string + assert "NORM layer" in cl_config_string + assert "zero" in cl_config_string + assert "avg_pool" in cl_config_string + + arch = resolved_pipeline.ARCH + arch_config_string = string_formatter.format_value(arch) + assert arch_config_string + # Check that arch contains CL-related operations (nested structure) + assert "Cell(" in arch_config_string + assert "Residual" in arch_config_string + assert "Sequential" in arch_config_string diff --git a/tests/test_neps_space/test_search_space__nos_like.py b/tests/test_neps_space/test_search_space__nos_like.py new file mode 100644 index 000000000..0de0cdabc --- /dev/null +++ b/tests/test_neps_space/test_search_space__nos_like.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import pytest + +from neps.space.neps_spaces import neps_space, string_formatter +from neps.space.neps_spaces.parameters import ( + ByName, + Categorical, + Integer, + Operation, + PipelineSpace, +) + + +class NosBench(PipelineSpace): + _UNARY_FUN = Categorical( + choices=( + Operation(operator="Square"), + Operation(operator="Exp"), + Operation(operator="Log"), + ) + ) + + _BINARY_FUN = Categorical( + choices=( + Operation(operator="Add"), + Operation(operator="Sub"), + Operation(operator="Mul"), + ) + ) + + _TERNARY_FUN = Categorical( + choices=( + Operation(operator="Interpolate"), + Operation(operator="Bias_Correct"), + ) + ) + + _PARAMS = Categorical( + choices=( + Operation(operator="Params"), + Operation(operator="Gradient"), + Operation(operator="Opt_Step"), + ) + ) + _CONST = Integer(3, 8) + _VAR = Integer(9, 19) + + _POINTER = Categorical( + choices=( + _PARAMS.resample(), + _CONST.resample(), + _VAR.resample(), + ), + ) + + _UNARY = Operation( + operator="Unary", + args=( + _UNARY_FUN.resample(), + _POINTER.resample(), + ), + ) + + _BINARY = Operation( + operator="Binary", + args=( + _BINARY_FUN.resample(), + _POINTER.resample(), + _POINTER.resample(), + ), + ) + + _TERNARY = Operation( + operator="Ternary", + args=( + _TERNARY_FUN.resample(), + _POINTER.resample(), + _POINTER.resample(), + _POINTER.resample(), + ), + ) + + _F_ARGS = Categorical( + choices=( + _UNARY.resample(), + _BINARY.resample(), + _TERNARY.resample(), + ), + ) + + _F = Operation( + operator="Function", + args=(_F_ARGS.resample(),), + kwargs={"var": _VAR.resample()}, + ) + + _L_ARGS = Categorical( + choices=( + (_F.resample(),), + ( + _F.resample(), + ByName("_L").resample(), + ), + ), + ) + + _L = Operation( + operator="Line_operator", + args=_L_ARGS.resample(), + ) + + P = Operation( + operator="Program", + args=(_L.resample(),), + ) + + +@pytest.mark.repeat(500) +def test_resolve(): + pipeline = NosBench() + + try: + resolved_pipeline, _ = neps_space.resolve(pipeline) + except RecursionError: + pytest.xfail("XFAIL due to too much recursion.") + raise + + p = resolved_pipeline.P + p_config_string = string_formatter.format_value(p) + assert p_config_string diff --git a/tests/test_neps_space/test_search_space__recursion.py b/tests/test_neps_space/test_search_space__recursion.py new file mode 100644 index 000000000..acca5f306 --- /dev/null +++ b/tests/test_neps_space/test_search_space__recursion.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence + +from neps.space.neps_spaces import neps_space +from neps.space.neps_spaces.parameters import ( + ByName, + Categorical, + Float, + Operation, + PipelineSpace, +) + + +class Model: + """An inner function that sums the values and multiplies the result by a factor. + This class can be recursively used in a search space to create nested models. + """ + + def __init__( + self, + inner_function: Callable[[Sequence[float]], float], + factor: float, + ): + """Initialize the model with an inner function and a factor.""" + self.inner_function = inner_function + self.factor = factor + + def __call__(self, values: Sequence[float]) -> float: + return self.factor * self.inner_function(values) + + +class Sum: + """A simple inner function that sums the values.""" + + def __call__(self, values: Sequence[float]) -> float: + return sum(values) + + +class DemoRecursiveOperationSpace(PipelineSpace): + # The way to sample `factor` values + _factor = Float(lower=0, upper=1) + + # Sum + _sum = Operation(operator=Sum) + + # Model + # Can recursively request itself as an arg. + # Will be equivalent to something like one of + # `Model(Sum(), factor=0.1)` + # `Model(Model(Sum(), factor=0.1), factor=0.1)` + # `Model(Model(Model(Sum(), factor=0.1), factor=0.1), factor=0.1)` + # ... + # If we want the `factor` values to be different, + # we just request a resample for them + _inner_function = Categorical( + choices=(_sum, ByName("model").resample()), + ) + model = Operation( + operator=Model, + args=(_inner_function.resample(),), + kwargs={"factor": _factor}, + ) + + +def test_recursion(): + pipeline = DemoRecursiveOperationSpace() + + # Across `n` iterations we collect the number of seen inner `Model` counts. + # We expect to see at least `k` cases for that number + expected_minimal_number_of_recursions = 3 + seen_inner_model_counts = [] + + for _ in range(200): + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + model = resolved_pipeline.model + assert model.operator is Model + + inner_function = model.args[0] + seen_factors = [model.kwargs["factor"]] + seen_inner_model_count = 0 + + # Loop into the inner operators until we have no more nested `Model` args + while inner_function.operator is Model: + seen_factors.append(inner_function.kwargs["factor"]) + seen_inner_model_count += 1 + inner_function = inner_function.args[0] + + # At this point we should have gone deep enough to have the terminal `Sum` + assert inner_function.operator is Sum + + # We should have seen as many factors as inner models + 1 for the outer one + assert len(seen_factors) == seen_inner_model_count + 1 + + # All the factors should be the same value (shared) + assert len(set(seen_factors)) == 1 + assert isinstance(seen_factors[0], float) + + # Add the number of seen `Model` operator in the loop + seen_inner_model_counts.append(seen_inner_model_count) + + assert len(set(seen_inner_model_counts)) >= expected_minimal_number_of_recursions + + +# TODO: test context with recursion (`samplings_to_make`) diff --git a/tests/test_neps_space/test_search_space__resampled.py b/tests/test_neps_space/test_search_space__resampled.py new file mode 100644 index 000000000..e5b4a5791 --- /dev/null +++ b/tests/test_neps_space/test_search_space__resampled.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +import pytest + +from neps.space.neps_spaces import neps_space +from neps.space.neps_spaces.parameters import ( + Categorical, + ConfidenceLevel, + Float, + Integer, + Operation, + PipelineSpace, +) + + +class ActPipelineSimpleFloat(PipelineSpace): + prelu_init_value = Float( + lower=0, + upper=1000000, + log=False, + prior=0.25, + prior_confidence=ConfidenceLevel.LOW, + ) + + prelu_shared1 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value}, + ) + prelu_shared2 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value}, + ) + + prelu_own_clone1 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value.resample()}, + ) + prelu_own_clone2 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value.resample()}, + ) + + _prelu_init_resampled = prelu_init_value.resample() + prelu_common_clone1 = Operation( + operator="prelu", + kwargs={"init": _prelu_init_resampled}, + ) + prelu_common_clone2 = Operation( + operator="prelu", + kwargs={"init": _prelu_init_resampled}, + ) + + +class ActPipelineComplexInteger(PipelineSpace): + prelu_init_value = Integer(lower=0, upper=1000000) + + prelu_shared1 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value}, + ) + prelu_shared2 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value}, + ) + + prelu_own_clone1 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value.resample()}, + ) + prelu_own_clone2 = Operation( + operator="prelu", + kwargs={"init": prelu_init_value.resample()}, + ) + + _prelu_init_resampled = prelu_init_value.resample() + prelu_common_clone1 = Operation( + operator="prelu", + kwargs={"init": _prelu_init_resampled}, + ) + prelu_common_clone2 = Operation( + operator="prelu", + kwargs={"init": _prelu_init_resampled}, + ) + + act: Operation = Operation( + operator="sequential6", + args=( + prelu_shared1, + prelu_shared2, + prelu_own_clone1, + prelu_own_clone2, + prelu_common_clone1, + prelu_common_clone2, + ), + kwargs={ + "prelu_shared": prelu_shared1, + "prelu_own_clone": prelu_own_clone1, + "prelu_common_clone": prelu_common_clone1, + "resampled_hp_value": prelu_init_value.resample(), + }, + ) + + +class CellPipelineCategorical(PipelineSpace): + conv_block = Categorical( + choices=( + Operation(operator="conv1"), + Operation(operator="conv2"), + ), + ) + + op1 = Categorical( + choices=( + conv_block, + Operation("op1"), + ), + ) + op2 = Categorical( + choices=( + conv_block.resample(), + Operation("op2"), + ), + ) + + _resampled_op1 = op1.resample() + cell = Operation( + operator="cell", + args=( + op1, + op2, + _resampled_op1, + op2.resample(), + _resampled_op1, + op2.resample(), + ), + ) + + +@pytest.mark.repeat(200) +def test_resampled_float(): + pipeline = ActPipelineSimpleFloat() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ( + "prelu_init_value", + "prelu_shared1", + "prelu_shared2", + "prelu_own_clone1", + "prelu_own_clone2", + "prelu_common_clone1", + "prelu_common_clone2", + ) + + prelu_init_value = resolved_pipeline.prelu_init_value + prelu_shared1 = resolved_pipeline.prelu_shared1.kwargs["init"] + prelu_shared2 = resolved_pipeline.prelu_shared2.kwargs["init"] + resampled_values = ( + resolved_pipeline.prelu_own_clone1.kwargs["init"], + resolved_pipeline.prelu_own_clone2.kwargs["init"], + resolved_pipeline.prelu_common_clone1.kwargs["init"], + resolved_pipeline.prelu_common_clone2.kwargs["init"], + ) + + assert isinstance(prelu_init_value, float) + assert isinstance(prelu_shared1, float) + assert isinstance(prelu_shared2, float) + assert all(isinstance(resampled_value, float) for resampled_value in resampled_values) + + assert prelu_init_value == prelu_shared1 + assert prelu_init_value == prelu_shared2 + + assert len(set(resampled_values)) == len(resampled_values) + assert all( + resampled_value != prelu_init_value for resampled_value in resampled_values + ) + + +@pytest.mark.repeat(200) +def test_resampled_integer(): + pipeline = ActPipelineComplexInteger() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ( + "prelu_init_value", + "prelu_shared1", + "prelu_shared2", + "prelu_own_clone1", + "prelu_own_clone2", + "prelu_common_clone1", + "prelu_common_clone2", + "act", + ) + + prelu_init_value = resolved_pipeline.prelu_init_value + prelu_shared1 = resolved_pipeline.prelu_shared1.kwargs["init"] + prelu_shared2 = resolved_pipeline.prelu_shared2.kwargs["init"] + resampled_values = ( + resolved_pipeline.prelu_own_clone1.kwargs["init"], + resolved_pipeline.prelu_own_clone2.kwargs["init"], + resolved_pipeline.prelu_common_clone1.kwargs["init"], + resolved_pipeline.prelu_common_clone2.kwargs["init"], + ) + + assert isinstance(prelu_init_value, int) + assert isinstance(prelu_shared1, int) + assert isinstance(prelu_shared2, int) + assert all(isinstance(resampled_value, int) for resampled_value in resampled_values) + + assert prelu_init_value == prelu_shared1 + assert prelu_init_value == prelu_shared2 + + assert len(set(resampled_values)) == len(resampled_values) + assert all( + resampled_value != prelu_init_value for resampled_value in resampled_values + ) + + act = resolved_pipeline.act + + act_args = tuple(op.kwargs["init"] for op in act.args) + sampled_values = (prelu_shared1, prelu_shared2, *resampled_values) + assert len(act_args) == len(sampled_values) + for act_arg, sampled_value in zip(act_args, sampled_values, strict=False): + assert act_arg is sampled_value + + act_resampled_prelu_shared = act.kwargs["prelu_shared"].kwargs["init"] + act_resampled_prelu_own_clone = act.kwargs["prelu_own_clone"].kwargs["init"] + act_resampled_prelu_common_clone = act.kwargs["prelu_common_clone"].kwargs["init"] + + assert isinstance(act_resampled_prelu_shared, int) + assert isinstance(act_resampled_prelu_own_clone, int) + assert isinstance(act_resampled_prelu_common_clone, int) + + assert act_resampled_prelu_shared == prelu_init_value + assert act_resampled_prelu_own_clone != prelu_init_value + assert act_resampled_prelu_common_clone != prelu_init_value + assert act_resampled_prelu_own_clone != act_resampled_prelu_common_clone + + act_resampled_hp_value = act.kwargs["resampled_hp_value"] + assert isinstance(act_resampled_hp_value, int) + assert act_resampled_hp_value != prelu_init_value + assert all( + resampled_value != act_resampled_hp_value for resampled_value in resampled_values + ) + + +@pytest.mark.repeat(200) +def test_resampled_categorical(): + pipeline = CellPipelineCategorical() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ( + "conv_block", + "op1", + "op2", + "cell", + ) + + conv_block = resolved_pipeline.conv_block + assert conv_block is not pipeline.conv_block + + op1 = resolved_pipeline.op1 + op2 = resolved_pipeline.op2 + assert op1 is not pipeline.op1 + assert op2 is not pipeline.op2 + + assert isinstance(op1, Operation) + assert isinstance(op2, Operation) + + assert (op1 is conv_block) or (op1.operator == "op1") + assert op2.operator in ("conv1", "conv2", "op2") + + cell = resolved_pipeline.cell + assert cell is not pipeline.cell + + cell_args1 = cell.args[0] + cell_args2 = cell.args[1] + + assert cell_args1 is op1 + assert cell_args2 is op2 + # todo: think about what more tests we can have for cell_args[3-6] diff --git a/tests/test_neps_space/test_search_space__reuse_arch_elements.py b/tests/test_neps_space/test_search_space__reuse_arch_elements.py new file mode 100644 index 000000000..6877e9a0d --- /dev/null +++ b/tests/test_neps_space/test_search_space__reuse_arch_elements.py @@ -0,0 +1,441 @@ +from __future__ import annotations + +import pytest + +import neps.space.neps_spaces.sampling +from neps.space.neps_spaces import neps_space, string_formatter +from neps.space.neps_spaces.parameters import ( + Categorical, + ConfidenceLevel, + Float, + Integer, + Operation, + PipelineSpace, +) + + +class ActPipelineSimple(PipelineSpace): + prelu_with_args = Operation( + operator="prelu_with_args", + args=(0.1, 0.2), + ) + prelu_with_kwargs = Operation( + operator="prelu_with_kwargs", + kwargs={"init": 0.1}, + ) + relu = Operation(operator="relu") + + act: Operation = Categorical( + choices=(prelu_with_args, prelu_with_kwargs, relu), + ) + + +class ActPipelineComplex(PipelineSpace): + prelu_init_value: float = Float(lower=0.1, upper=0.9) + prelu = Operation( + operator="prelu", + kwargs={"init": prelu_init_value}, + ) + act: Operation = Categorical( + choices=(prelu,), + ) + + +class FixedPipeline(PipelineSpace): + prelu_init_value: float = 0.5 + prelu = Operation( + operator="prelu", + kwargs={"init": prelu_init_value}, + ) + act = prelu + + +_conv_choices_low = ("conv1x1", "conv3x3") +_conv_choices_high = ("conv5x5", "conv9x9") +_conv_choices_prior_confidence_choices = ( + ConfidenceLevel.LOW, + ConfidenceLevel.MEDIUM, + ConfidenceLevel.HIGH, +) + + +class ConvPipeline(PipelineSpace): + conv_choices_prior_index: int = Integer( + lower=0, + upper=1, + log=False, + prior=0, + prior_confidence=ConfidenceLevel.LOW, + ) + conv_choices_prior_confidence: ConfidenceLevel = Categorical( + choices=_conv_choices_prior_confidence_choices, + prior=1, + prior_confidence=ConfidenceLevel.LOW, + ) + conv_choices: tuple[str, ...] = Categorical( + choices=(_conv_choices_low, _conv_choices_high), + prior=conv_choices_prior_index, + prior_confidence=conv_choices_prior_confidence, + ) + + _conv1: str = Categorical( + choices=conv_choices, + ) + _conv2: str = Categorical( + choices=conv_choices, + ) + + conv_block: Operation = Categorical( + choices=( + Operation( + operator="sequential3", + args=[_conv1, _conv2, _conv1], + ), + ), + ) + + +class CellPipeline(PipelineSpace): + _act = Operation(operator="relu") + _conv = Operation(operator="conv3x3") + _norm = Operation(operator="batch") + + conv_block = Operation(operator="sequential3", args=(_act, _conv, _norm)) + + op1 = Categorical( + choices=( + conv_block, + Operation(operator="zero"), + Operation(operator="avg_pool"), + ), + ) + op2 = Categorical( + choices=( + conv_block, + Operation(operator="zero"), + Operation(operator="avg_pool"), + ), + ) + + _some_int = 2 + _some_float = Float(lower=0.5, upper=0.5) + + cell = Operation( + operator="cell", + args=(op1, op2, op1, op2, op1, op2), + kwargs={"float_hp": _some_float, "int_hp": _some_int}, + ) + + +@pytest.mark.repeat(50) +def test_nested_simple(): + pipeline = ActPipelineSimple() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ( + "prelu_with_args", + "prelu_with_kwargs", + "relu", + "act", + ) + + assert resolved_pipeline.prelu_with_kwargs is pipeline.prelu_with_kwargs + assert resolved_pipeline.prelu_with_args is pipeline.prelu_with_args + assert resolved_pipeline.relu is pipeline.relu + + assert resolved_pipeline.act in ( + resolved_pipeline.prelu_with_kwargs, + resolved_pipeline.prelu_with_args, + resolved_pipeline.relu, + ) + + +@pytest.mark.repeat(50) +def test_nested_simple_string(): + # Format is now always expanded, check for content + pipeline = ActPipelineSimple() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + act = resolved_pipeline.act + act_config_string = string_formatter.format_value(act) + assert act_config_string + + # Check for one of the possible operations + is_relu = "relu" in act_config_string.lower() + is_prelu_args = "prelu_with_args" in act_config_string and "0.1" in act_config_string + is_prelu_kwargs = ( + "prelu_with_kwargs" in act_config_string and "init=0.1" in act_config_string + ) + assert is_relu or is_prelu_args or is_prelu_kwargs + + +@pytest.mark.repeat(50) +def test_nested_complex(): + pipeline = ActPipelineComplex() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ( + "prelu_init_value", + "prelu", + "act", + ) + + prelu_init_value = resolved_pipeline.prelu_init_value + assert 0.1 <= prelu_init_value <= 0.9 + + prelu = resolved_pipeline.prelu + assert prelu.operator == "prelu" + assert isinstance(prelu.kwargs["init"], float) + assert prelu.kwargs["init"] is prelu_init_value + assert not prelu.args + + act = resolved_pipeline.act + assert act.operator == "prelu" + assert act is prelu + + +@pytest.mark.repeat(50) +def test_nested_complex_string(): + pipeline = ActPipelineComplex() + + resolved_pipeline, _ = neps_space.resolve(pipeline) + + act = resolved_pipeline.act + act_config_string = string_formatter.format_value(act) + assert act_config_string + + # Format is now expanded, check for content + assert "prelu" in act_config_string + assert "init=" in act_config_string + # Extract the init value (should be between 0.1 and 0.9) + import re + + match = re.search(r"init=([\d.]+)", act_config_string) + assert match is not None + init_value = float(match.group(1)) + assert 0.1 <= init_value <= 0.9 + + +def test_fixed_pipeline(): + pipeline = FixedPipeline() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == tuple( + pipeline.get_attrs().keys() + ) + + assert resolved_pipeline.prelu_init_value == pipeline.prelu_init_value + assert resolved_pipeline.prelu is pipeline.prelu + assert resolved_pipeline.act is pipeline.act + assert resolved_pipeline is pipeline + + +def test_fixed_pipeline_string(): + pipeline = FixedPipeline() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + act = resolved_pipeline.act + act_config_string = string_formatter.format_value(act) + assert act_config_string + # Check content rather than exact format (now always expanded) + assert "prelu" in act_config_string + assert "init=0.5" in act_config_string + + +@pytest.mark.repeat(50) +def test_simple_reuse(): + pipeline = ConvPipeline() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ( + "conv_choices_prior_index", + "conv_choices_prior_confidence", + "conv_choices", + "conv_block", + ) + + conv_choices_prior_index = resolved_pipeline.conv_choices_prior_index + assert conv_choices_prior_index in (0, 1) + + conv_choices_prior_confidence = resolved_pipeline.conv_choices_prior_confidence + assert conv_choices_prior_confidence in _conv_choices_prior_confidence_choices + + conv_choices = resolved_pipeline.conv_choices + assert conv_choices in (_conv_choices_low, _conv_choices_high) + + conv_block = resolved_pipeline.conv_block + assert conv_block.operator == "sequential3" + for conv in conv_block.args: + assert conv in conv_choices + assert conv_block.args[0] == conv_block.args[2] + + +@pytest.mark.repeat(50) +def test_simple_reuse_string(): + # Check that the formatted string reflects the reuse pattern correctly + # Format is now always expanded, so check semantic content + pipeline = ConvPipeline() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + conv_block = resolved_pipeline.conv_block + conv_block_config_string = string_formatter.format_value(conv_block) + assert conv_block_config_string + + # Should contain sequential3 and three conv operations + assert "sequential3" in conv_block_config_string + assert conv_block_config_string.count("conv") == 3 + + # Extract the three conv operations - they should follow the reuse pattern + # where first and third are the same + import re + + convs = re.findall(r"(conv\dx\d)", conv_block_config_string) + assert len(convs) == 3 + assert convs[0] == convs[2], f"First and third conv should match: {convs}" + + +@pytest.mark.repeat(50) +def test_shared_complex(): + pipeline = CellPipeline() + + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + assert resolved_pipeline is not pipeline + assert resolved_pipeline is not None + assert tuple(resolved_pipeline.get_attrs().keys()) == ( + "conv_block", + "op1", + "op2", + "cell", + ) + + conv_block = resolved_pipeline.conv_block + assert conv_block is pipeline.conv_block + + op1 = resolved_pipeline.op1 + op2 = resolved_pipeline.op2 + assert op1 is not pipeline.op1 + assert op2 is not pipeline.op2 + assert isinstance(op1, Operation) + assert isinstance(op2, Operation) + + if op1 is op2: + assert op1 is conv_block + else: + assert op1.operator in {"zero", "avg_pool", "sequential3"} + assert op2.operator in {"zero", "avg_pool", "sequential3"} + if op1.operator == "sequential3" or op2.operator == "sequential3": + assert op1.operator != op2.operator + + cell = resolved_pipeline.cell + assert cell is not pipeline.cell + assert cell.operator == "cell" + assert cell.args[0] is op1 + assert cell.args[1] is op2 + assert cell.args[2] is op1 + assert cell.args[3] is op2 + assert cell.args[4] is op1 + assert cell.args[5] is op2 + assert len(cell.kwargs) == 2 + assert cell.kwargs["float_hp"] == 0.5 + assert cell.kwargs["int_hp"] == 2 + + +@pytest.mark.repeat(50) +def test_shared_complex_string(): + # The new formatter outputs all operations in full, rather than using + # references for shared operations. Check for key elements instead of exact format. + + pipeline = CellPipeline() + resolved_pipeline, _resolution_context = neps_space.resolve(pipeline) + + cell = resolved_pipeline.cell + cell_config_string = string_formatter.format_value(cell) + + # Verify essential elements are present + assert cell_config_string + assert cell_config_string.startswith("cell(") + assert "float_hp=0.5" in cell_config_string + assert "int_hp=2" in cell_config_string + + # Check that the operation types that could appear are present + # (at least one of avg_pool, zero, or sequential3 should appear) + has_operation = ( + "avg_pool()" in cell_config_string + or "zero()" in cell_config_string + or "sequential3(" in cell_config_string + ) + assert has_operation + + +def test_shared_complex_context(): + # todo: move the context testing part to its own test file. + # This one should only do the reuse tests + + # todo: add a more complex test, where we have hidden Categorical choices. + # E.g. add Resample along the way + + samplings_to_make = { + "Resolvable.op1::categorical__3": 2, + "Resolvable.op2::categorical__3": 1, + "Resolvable.cell.kwargs.mapping_value{float_hp}::float__0.5_0.5_False": 0.5, + } + + pipeline = CellPipeline() + + resolved_pipeline_first, _resolution_context_first = neps_space.resolve( + pipeline=pipeline, + domain_sampler=neps.space.neps_spaces.sampling.OnlyPredefinedValuesSampler( + predefined_samplings=samplings_to_make, + ), + ) + sampled_values_first = _resolution_context_first.samplings_made + + assert resolved_pipeline_first is not pipeline + assert sampled_values_first is not None + assert sampled_values_first is not samplings_to_make + assert sampled_values_first == samplings_to_make + assert list(sampled_values_first.items()) == list(samplings_to_make.items()) + + resolved_pipeline_second, _resolution_context_second = neps_space.resolve( + pipeline=pipeline, + domain_sampler=neps.space.neps_spaces.sampling.OnlyPredefinedValuesSampler( + predefined_samplings=samplings_to_make, + ), + ) + sampled_values_second = _resolution_context_second.samplings_made + + assert resolved_pipeline_second is not pipeline + assert resolved_pipeline_second is not None + assert sampled_values_second is not samplings_to_make + assert sampled_values_second == samplings_to_make + assert list(sampled_values_second.items()) == list(samplings_to_make.items()) + + # the second resolution should give us a new object + assert resolved_pipeline_second is not resolved_pipeline_first + + # The new formatter outputs operations in full rather than using references. + # Check that both resolutions produce the same format and contain expected operations. + config_str_first = string_formatter.format_value(resolved_pipeline_first.cell) + config_str_second = string_formatter.format_value(resolved_pipeline_second.cell) + + # Both resolutions with same samplings should produce identical output + assert config_str_first == config_str_second + + # Check essential elements are present + assert config_str_first.startswith("cell(") + assert "avg_pool()" in config_str_first + assert "zero()" in config_str_first + assert "float_hp=0.5" in config_str_first + assert "int_hp=2" in config_str_first diff --git a/tests/test_neps_space/test_space_conversion_and_compatibility.py b/tests/test_neps_space/test_space_conversion_and_compatibility.py new file mode 100644 index 000000000..1309690d8 --- /dev/null +++ b/tests/test_neps_space/test_space_conversion_and_compatibility.py @@ -0,0 +1,491 @@ +"""Tests for space conversion and algorithm compatibility in NePS.""" + +from __future__ import annotations + +import pytest + +import neps +from neps.optimizers import algorithms +from neps.space.neps_spaces.neps_space import ( + check_neps_space_compatibility, + convert_classic_to_neps_search_space, + convert_neps_to_classic_search_space, +) +from neps.space.neps_spaces.parameters import ( + Categorical, + ConfidenceLevel, + Fidelity, + Float, + Integer, + IntegerFidelity, + Operation, + PipelineSpace, +) + + +class SimpleHPOSpace(PipelineSpace): + """Simple hyperparameter-only space that can be converted to classic.""" + + x = Float(lower=0.0, upper=1.0, prior=0.5, prior_confidence=ConfidenceLevel.MEDIUM) + y = Integer(lower=1, upper=10, prior=5, prior_confidence=ConfidenceLevel.HIGH) + z = Categorical( + choices=("a", "b", "c"), prior=1, prior_confidence=ConfidenceLevel.LOW + ) + + +class SimpleHPOWithFidelitySpace(PipelineSpace): + """Simple hyperparameter space with fidelity.""" + + x = Float(lower=0.0, upper=1.0, prior=0.5, prior_confidence=ConfidenceLevel.MEDIUM) + y = Integer(lower=1, upper=10, prior=5, prior_confidence=ConfidenceLevel.HIGH) + epochs = IntegerFidelity(lower=1, upper=100) + + +class ComplexNepsSpace(PipelineSpace): + """Complex NePS space that cannot be converted to classic.""" + + # Basic parameters + factor = Float( + lower=0.1, upper=2.0, prior=1.0, prior_confidence=ConfidenceLevel.MEDIUM + ) + + # Operation with resampled parameters + operation = Operation( + operator=lambda x, y: x * y, + args=(factor, factor.resample()), + ) + + # Categorical with operations as choices + choice = Categorical( + choices=(operation, factor), + prior=0, + prior_confidence=ConfidenceLevel.LOW, + ) + + +# ===== Test space conversion functions ===== + + +def test_convert_classic_to_neps(): + """Test conversion from classic SearchSpace to NePS PipelineSpace.""" + # Create a classic SearchSpace with various parameter types + classic_space = neps.SearchSpace( + { + "float_param": neps.HPOFloat(0.0, 1.0, prior=0.5, prior_confidence="medium"), + "int_param": neps.HPOInteger(1, 10, prior=5, prior_confidence="high"), + "cat_param": neps.HPOCategorical( + ["a", "b", "c"], prior="b", prior_confidence="low" + ), + "fidelity_param": neps.HPOInteger(1, 100, is_fidelity=True), + "constant_param": neps.HPOConstant("constant_value"), + } + ) + + # Convert to NePS space + neps_space = convert_classic_to_neps_search_space(classic_space) + assert isinstance(neps_space, PipelineSpace) + + # Verify attributes are preserved + neps_attrs = neps_space.get_attrs() + assert len(neps_attrs) == 5 + assert all( + name in neps_attrs + for name in [ + "float_param", + "int_param", + "cat_param", + "fidelity_param", + "constant_param", + ] + ) + + # Verify types and properties + assert isinstance(neps_attrs["float_param"], Float) + assert neps_attrs["float_param"].has_prior + assert neps_attrs["float_param"].prior == 0.5 + assert neps_attrs["float_param"].prior_confidence == ConfidenceLevel.MEDIUM + + assert isinstance(neps_attrs["int_param"], Integer) + assert neps_attrs["int_param"].has_prior + assert neps_attrs["int_param"].prior == 5 + assert neps_attrs["int_param"].prior_confidence == ConfidenceLevel.HIGH + + assert isinstance(neps_attrs["cat_param"], Categorical) + assert neps_attrs["cat_param"].has_prior + assert neps_attrs["cat_param"].prior == 1 # Index of "b" in choices + assert neps_attrs["cat_param"].prior_confidence == ConfidenceLevel.LOW + + assert isinstance(neps_attrs["fidelity_param"], Fidelity) + assert isinstance(neps_attrs["fidelity_param"].domain, Integer) + + # Constant should be preserved as-is + assert neps_attrs["constant_param"] == "constant_value" + + +def test_convert_neps_to_classic_simple(): + """Test conversion from simple NePS PipelineSpace to classic SearchSpace.""" + space = SimpleHPOSpace() + + # Convert to classic space + classic_space = convert_neps_to_classic_search_space(space) + assert classic_space is not None + assert isinstance(classic_space, neps.SearchSpace) + + # Verify attributes are preserved + classic_attrs = classic_space.elements + assert len(classic_attrs) == 3 + assert all(name in classic_attrs for name in ["x", "y", "z"]) + + # Verify types and priors + x_param = classic_attrs["x"] + assert isinstance(x_param, neps.HPOFloat) + assert x_param.lower == 0.0 + assert x_param.upper == 1.0 + assert x_param.prior == 0.5 + assert x_param.prior_confidence == "medium" + + y_param = classic_attrs["y"] + assert isinstance(y_param, neps.HPOInteger) + assert y_param.lower == 1 + assert y_param.upper == 10 + assert y_param.prior == 5 + assert y_param.prior_confidence == "high" + + z_param = classic_attrs["z"] + assert isinstance(z_param, neps.HPOCategorical) + assert set(z_param.choices) == {"a", "b", "c"} # Order might vary + assert z_param.prior == "b" + assert z_param.prior_confidence == "low" + + +def test_convert_neps_to_classic_with_fidelity(): + """Test conversion from NePS PipelineSpace with fidelity to classic SearchSpace.""" + space = SimpleHPOWithFidelitySpace() + + # Convert to classic space + classic_space = convert_neps_to_classic_search_space(space) + assert classic_space is not None + assert isinstance(classic_space, neps.SearchSpace) + + # Verify fidelity parameter + epochs_param = classic_space.elements["epochs"] + assert isinstance(epochs_param, neps.HPOInteger) + assert epochs_param.is_fidelity + assert epochs_param.lower == 1 + assert epochs_param.upper == 100 + + +def test_convert_complex_neps_to_classic_fails(): + """Test that complex NePS spaces cannot be converted to classic.""" + space = ComplexNepsSpace() + + # This space should NOT be convertible to classic + converted = convert_neps_to_classic_search_space(space) + assert converted is None + + +def test_round_trip_conversion(): + """Test that simple spaces can be converted back and forth.""" + # Start with classic space + original_classic = neps.SearchSpace( + { + "x": neps.HPOFloat(0.0, 1.0, prior=0.5, prior_confidence="medium"), + "y": neps.HPOInteger(1, 10, prior=5, prior_confidence="high"), + "z": neps.HPOCategorical(["a", "b", "c"], prior="b", prior_confidence="low"), + } + ) + + # Convert to NePS and back + neps_space = convert_classic_to_neps_search_space(original_classic) + converted_back = convert_neps_to_classic_search_space(neps_space) + + assert converted_back is not None + assert len(converted_back.elements) == len(original_classic.elements) + + # Verify parameters are equivalent + for name in original_classic.elements: + original_param = original_classic.elements[name] + converted_param = converted_back.elements[name] + + assert type(original_param) is type(converted_param) + + # Check bounds for numerical parameters + if isinstance(original_param, neps.HPOFloat | neps.HPOInteger): + assert original_param.lower == converted_param.lower + assert original_param.upper == converted_param.upper + + # Check choices for categorical parameters + if isinstance(original_param, neps.HPOCategorical): + # Sort choices for comparison since order might differ + assert set(original_param.choices) == set(converted_param.choices) + + # Check priors + if hasattr(original_param, "prior") and hasattr(converted_param, "prior"): + assert original_param.prior == converted_param.prior + + +# ===== Test algorithm compatibility ===== + + +def test_neps_only_algorithms(): + """Test that NePS-only algorithms are correctly identified.""" + neps_only_algorithms = [ + algorithms.neps_random_search, + algorithms.neps_hyperband, + algorithms.complex_random_search, + algorithms.neps_priorband, + ] + + for algo in neps_only_algorithms: + compatibility = check_neps_space_compatibility(algo) + assert compatibility in [ + "neps", + "both", + ], f"Algorithm {algo.__name__} should be neps or both compatible" + + +def test_classic_and_both_algorithms(): + """Test that classic algorithms that work with both spaces are correctly identified.""" + both_compatible_algorithms = [ + algorithms.random_search, + algorithms.hyperband, + algorithms.priorband, + ] + + for algo in both_compatible_algorithms: + compatibility = check_neps_space_compatibility(algo) + assert compatibility in [ + "classic", + "both", + ], f"Algorithm {algo.__name__} should be classic or both compatible" + + +def test_algorithm_compatibility_with_string_names(): + """Test algorithm compatibility checking with string names.""" + # Note: String-based compatibility checking may not be fully implemented + # Test with actual algorithm functions instead + + # Test NePS-only algorithms + neps_only_algorithms = [ + algorithms.neps_random_search, + algorithms.neps_hyperband, + algorithms.complex_random_search, + ] + + for algo in neps_only_algorithms: + compatibility = check_neps_space_compatibility(algo) + assert compatibility in [ + "neps", + "both", + ], f"Algorithm {algo.__name__} should be neps or both compatible" + + # Test classic/both algorithms + classic_algorithms = [ + algorithms.random_search, + algorithms.hyperband, + ] + + for algo in classic_algorithms: + compatibility = check_neps_space_compatibility(algo) + assert compatibility in [ + "classic", + "both", + ], f"Algorithm {algo.__name__} should be classic or both compatible" + + +def test_algorithm_compatibility_with_tuples(): + """Test algorithm compatibility checking with tuple configurations.""" + # Test with tuple configuration + neps_config = ("neps_random_search", {"ignore_fidelity": True}) + compatibility = check_neps_space_compatibility(neps_config) + assert compatibility in ["neps", "both"] + + classic_config = ("random_search", {"some_param": "value"}) + compatibility = check_neps_space_compatibility(classic_config) + assert compatibility in ["classic", "both"] + + +def test_auto_algorithm_compatibility(): + """Test that 'auto' algorithm is handled correctly.""" + compatibility = check_neps_space_compatibility("auto") + assert compatibility == "both" + + +# ===== Test NePS hyperband specific functionality ===== + + +def test_neps_hyperband_requires_fidelity(): + """Test that neps_hyperband requires fidelity parameters.""" + # Space without fidelity should fail + space_no_fidelity = SimpleHPOSpace() + + with pytest.raises((ValueError, AssertionError)): + algorithms.neps_hyperband(pipeline_space=space_no_fidelity) + + +def test_neps_hyperband_accepts_fidelity_space(): + """Test that neps_hyperband accepts spaces with fidelity.""" + space_with_fidelity = SimpleHPOWithFidelitySpace() + + # Should not raise an error + optimizer = algorithms.neps_hyperband(pipeline_space=space_with_fidelity) + assert optimizer is not None + + +def test_neps_hyperband_rejects_classic_space(): + """Test that neps_hyperband rejects classic SearchSpace.""" + # Type system should prevent this at compile time + # Instead, test that type checking works as expected + + # Create a proper NePS space that should work + class TestSpace(PipelineSpace): + x = Float(0.0, 1.0) + epochs = IntegerFidelity(1, 100) + + space = TestSpace() + + # This should work fine with proper NePS space + optimizer = algorithms.neps_hyperband(pipeline_space=space, eta=3) + assert optimizer is not None + + +@pytest.mark.parametrize("eta", [2, 3, 4, 5]) +def test_neps_hyperband_eta_values(eta): + """Test neps_hyperband with different eta values.""" + space = SimpleHPOWithFidelitySpace() + optimizer = algorithms.neps_hyperband(pipeline_space=space, eta=eta) + assert optimizer is not None + + +@pytest.mark.parametrize("sampler", ["uniform", "prior"]) +def test_neps_hyperband_samplers(sampler): + """Test neps_hyperband with different samplers.""" + space = SimpleHPOWithFidelitySpace() + optimizer = algorithms.neps_hyperband(pipeline_space=space, sampler=sampler) + assert optimizer is not None + + +@pytest.mark.parametrize("sample_prior_first", [False, True, "highest_fidelity"]) +def test_neps_hyperband_sample_prior_first(sample_prior_first): + """Test neps_hyperband with different sample_prior_first options.""" + space = SimpleHPOWithFidelitySpace() + optimizer = algorithms.neps_hyperband( + pipeline_space=space, sample_prior_first=sample_prior_first + ) + assert optimizer is not None + + +# ===== Test space compatibility with different optimizers ===== + + +def test_simple_space_works_with_both_optimizers(): + """Test that simple HPO spaces work with both classic and NePS optimizers.""" + space = SimpleHPOSpace() + + # Should work with NePS-only optimizers + neps_optimizer = algorithms.neps_random_search(pipeline_space=space) + assert neps_optimizer is not None + + # Should also be convertible and work with classic optimizers + converted_space = convert_neps_to_classic_search_space(space) + assert converted_space is not None + + classic_optimizer = algorithms.random_search( + pipeline_space=converted_space, use_priors=True + ) + assert classic_optimizer is not None + + +def test_complex_space_only_works_with_neps_optimizers(): + """Test that complex NePS spaces only work with NePS-compatible optimizers.""" + space = ComplexNepsSpace() + + # Should work with NePS optimizers + neps_optimizer = algorithms.neps_random_search(pipeline_space=space) + assert neps_optimizer is not None + + # Should NOT be convertible to classic + converted_space = convert_neps_to_classic_search_space(space) + assert converted_space is None + + +def test_fidelity_space_compatibility(): + """Test fidelity space compatibility with different optimizers.""" + space = SimpleHPOWithFidelitySpace() + + # Should work with neps_hyperband (requires fidelity) + hyperband_optimizer = algorithms.neps_hyperband(pipeline_space=space) + assert hyperband_optimizer is not None + + # Should also work with other NePS optimizers (but need to ignore fidelity) + random_optimizer = algorithms.neps_random_search( + pipeline_space=space, ignore_fidelity=True + ) + assert random_optimizer is not None + + # Should be convertible to classic for non-neps-specific algorithms + converted_space = convert_neps_to_classic_search_space(space) + assert converted_space is not None + + # Classic hyperband should work with converted space + classic_hyperband = algorithms.hyperband(pipeline_space=converted_space) + assert classic_hyperband is not None + + +# ===== Edge cases and error handling ===== + + +def test_conversion_preserves_log_scaling(): + """Test that log scaling is preserved during conversion.""" + classic_space = neps.SearchSpace( + { + "log_param": neps.HPOFloat(1e-5, 1e-1, log=True), + } + ) + + neps_space = convert_classic_to_neps_search_space(classic_space) + # Access the Float parameter and check if it has a log attribute + log_param_neps = neps_space.get_attrs()["log_param"] + assert hasattr(log_param_neps, "log") + assert log_param_neps.log is True + + # Round-trip conversion should now preserve log scaling + converted_back = convert_neps_to_classic_search_space(neps_space) + assert converted_back is not None + # Check the log property specifically for float parameters + log_param = converted_back.elements["log_param"] + assert isinstance(log_param, neps.HPOFloat) + assert log_param.log is True + + +def test_conversion_handles_missing_priors(): + """Test that conversion works correctly when priors are missing.""" + classic_space = neps.SearchSpace( + { + "no_prior": neps.HPOFloat(0.0, 1.0), # No prior specified + } + ) + + neps_space = convert_classic_to_neps_search_space(classic_space) + param = neps_space.get_attrs()["no_prior"] + assert not param.has_prior + + converted_back = convert_neps_to_classic_search_space(neps_space) + assert converted_back is not None + # Check the prior property specifically for float parameters + no_prior_param = converted_back.elements["no_prior"] + assert isinstance(no_prior_param, neps.HPOFloat) + assert no_prior_param.prior is None + + +def test_conversion_handles_empty_spaces(): + """Test that conversion handles edge cases gracefully.""" + # Empty classic space + empty_classic = neps.SearchSpace({}) + neps_space = convert_classic_to_neps_search_space(empty_classic) + assert len(neps_space.get_attrs()) == 0 + + # Convert back + converted_back = convert_neps_to_classic_search_space(neps_space) + assert converted_back is not None + assert len(converted_back.elements) == 0 diff --git a/tests/test_neps_space/test_string_formatter.py b/tests/test_neps_space/test_string_formatter.py new file mode 100644 index 000000000..ac822e01b --- /dev/null +++ b/tests/test_neps_space/test_string_formatter.py @@ -0,0 +1,493 @@ +"""Comprehensive tests for string_formatter module.""" + +from __future__ import annotations + +import neps +from neps.space.neps_spaces.parameters import Operation +from neps.space.neps_spaces.string_formatter import ( + FormatterStyle, + format_value, +) + + +def test_simple_operation_no_args(): + """Test formatting an operation with no arguments - default shows ().""" + op = Operation(operator="ReLU") + result = format_value(op) + assert result == "ReLU()" + + +def test_simple_operation_no_args_with_parens(): + """Test formatting with show_empty_args=False to hide ().""" + op = Operation(operator="ReLU") + style = FormatterStyle(show_empty_args=False) + result = format_value(op, 0, style) + assert result == "ReLU" + + +def test_operation_with_args_only(): + """Test formatting an operation with positional args only - always expanded.""" + op = Operation(operator="Add", args=(1, 2, 3)) + result = format_value(op) + expected = """Add( + 1, + 2, + 3, +)""" + assert result == expected + + +def test_operation_with_kwargs_only(): + """Test formatting an operation with keyword args only - always expanded.""" + op = Operation(operator="Conv2d", kwargs={"in_channels": 3, "out_channels": 64}) + result = format_value(op) + expected = """Conv2d( + in_channels=3, + out_channels=64, +)""" + assert result == expected + + +def test_operation_with_args_and_kwargs(): + """Test formatting with both positional and keyword arguments - always expanded.""" + op = Operation( + operator="LinearLayer", + args=(128,), + kwargs={"activation": "relu", "dropout": 0.5}, + ) + result = format_value(op) + expected = """LinearLayer( + 128, + activation=relu, + dropout=0.5, +)""" + assert result == expected + + +def test_nested_operations(): + """Test formatting nested operations.""" + inner = Operation(operator="ReLU") + outer = Operation(operator="Sequential", args=(inner,)) + result = format_value(outer) + expected = """Sequential( + ReLU(), +)""" + assert result == expected + + +def test_deeply_nested_operations(): + """Test formatting deeply nested operations - all ops expanded.""" + conv = Operation( + operator="Conv2d", + kwargs={"in_channels": 3, "out_channels": 64, "kernel_size": 3}, + ) + relu = Operation(operator="ReLU") + pool = Operation(operator="MaxPool2d", kwargs={"kernel_size": 2}) + + sequential = Operation(operator="Sequential", args=(conv, relu, pool)) + + result = format_value(sequential) + expected = """Sequential( + Conv2d( + in_channels=3, + out_channels=64, + kernel_size=3, + ), + ReLU(), + MaxPool2d( + kernel_size=2, + ), +)""" + assert result == expected + + +def test_list_as_arg(): + """Test formatting with a list as an argument.""" + op = Operation(operator="Conv2d", kwargs={"kernel_size": [3, 4]}) + result = format_value(op) + expected = """Conv2d( + kernel_size=[3, 4], +)""" + assert result == expected + + +def test_long_list_as_arg(): + """Test formatting with a longer list that spans multiple lines.""" + long_list = list(range(20)) + op = Operation(operator="SomeOp", kwargs={"values": long_list}) + result = format_value(op) + + # Should have the list expanded + assert "values=[" in result + assert "0, 1, 2" in result # Multiple items per line + assert "]" in result + + +def test_tuple_as_arg(): + """Test formatting with a tuple as an argument.""" + op = Operation(operator="Shape", args=((64, 64, 3),)) + result = format_value(op) + expected = """Shape( + (64, 64, 3), +)""" + assert result == expected + + +def test_dict_as_kwarg(): + """Test formatting with a dict as a keyword argument value.""" + op = Operation( + operator="ConfigOp", + kwargs={"config": {"learning_rate": 0.001, "batch_size": 32}}, + ) + result = format_value(op) + # Dict gets expanded due to length + expected = """ConfigOp( + config={ + 'learning_rate': 0.001, + 'batch_size': 32, + }, +)""" + assert result == expected + + +def test_operations_in_list(): + """Test formatting operations inside a list argument - all ops expanded.""" + op1 = Operation(operator="Conv2d", kwargs={"channels": 32}) + op2 = Operation(operator="Conv2d", kwargs={"channels": 64}) + + container = Operation(operator="ModuleList", args=([op1, op2],)) + + result = format_value(container) + expected = """ModuleList( + [ + Conv2d( + channels=32, + ), + Conv2d( + channels=64, + ), + ], +)""" + assert result == expected + + +def test_operations_in_list_as_kwarg(): + """Test formatting operations inside a list that is a kwarg value.""" + op1 = Operation(operator="ReLU") + op2 = Operation(operator="Sigmoid") + + container = Operation(operator="Container", kwargs={"layers": [op1, op2]}) + + result = format_value(container) + expected = """Container( + layers=[ + ReLU(), + Sigmoid(), + ], +)""" + assert result == expected + + +def test_mixed_types_in_list(): + """Test formatting a list with mixed types including operations.""" + op = Operation(operator="ReLU") + mixed_list = [1, "hello", 3.14, op, [1, 2, 3]] + + container = Operation(operator="MixedContainer", args=(mixed_list,)) + + result = format_value(container) + + # Check that all elements are present + assert "1," in result + assert "hello," in result # Identifiers don't get quotes + assert "3.14," in result + assert "ReLU()," in result + assert "[1, 2, 3]," in result + + +def test_string_values_with_quotes(): + """Test that string values are properly quoted.""" + op = Operation( + operator="TextOp", + kwargs={ + "text": "hello world", + "quote_test": "it's a test", + "double_quotes": 'say "hello"', + }, + ) + result = format_value(op) + + # Check strings are properly represented + assert "text='hello world'" in result or 'text="hello world"' in result + assert "quote_test" in result + assert "double_quotes" in result + + +def test_complex_nested_structure(): + """Test a complex nested structure with all types.""" + # Build a complex structure + + conv = Operation( + operator="Conv2d", + kwargs={"in_channels": 3, "out_channels": 64, "kernel_size": [3, 4]}, + ) + relu = Operation(operator="ReLU") + + seq = Operation( + operator="Sequential", + args=([conv, relu],), + kwargs={"dropout": 0.5, "config": {"layers": [3, 64, 128]}}, + ) + + result = format_value(seq) + + # Verify structure + assert "Sequential(" in result + assert "Conv2d(" in result + assert "in_channels=3" in result + assert "kernel_size=[3, 4]" in result + assert "ReLU()," in result + assert "dropout=0.5" in result + assert "config=" in result + assert "'layers': [3, 64, 128]" in result + + +def test_non_operation_value(): + """Test formatting a non-Operation value.""" + # Should work with any value + result1 = format_value(42) + assert result1 == "42" + + result2 = format_value("hello") + assert result2 == "hello" # Identifiers don't get quotes + + result3 = format_value([1, 2, 3]) + assert result3 == "[1, 2, 3]" + + +def test_custom_indent(): + """Test using a custom indentation style - all ops expanded.""" + op = Operation(operator="Conv2d", kwargs={"channels": 64}) + style = FormatterStyle(indent_str=" ") # 4 spaces + + result = format_value(op, 0, style) + expected = """Conv2d( + channels=64, +)""" + assert result == expected + + +def test_empty_list(): + """Test formatting with empty list.""" + op = Operation(operator="Op", kwargs={"items": []}) + result = format_value(op) + expected = """Op( + items=[], +)""" + assert result == expected + + +def test_empty_tuple(): + """Test formatting with empty tuple.""" + op = Operation(operator="Op", args=((),)) + result = format_value(op) + expected = """Op( + (), +)""" + assert result == expected + + +def test_empty_dict(): + """Test formatting with empty dict.""" + op = Operation(operator="Op", kwargs={"config": {}}) + result = format_value(op) + expected = """Op( + config={}, +)""" + assert result == expected + + +def test_boolean_values(): + """Test formatting with boolean values - always expanded.""" + op = Operation(operator="Op", kwargs={"enabled": True, "debug": False, "count": 0}) + result = format_value(op) + expected = """Op( + enabled=True, + debug=False, + count=0, +)""" + assert result == expected + + +def test_none_value(): + """Test formatting with None value - always expanded.""" + op = Operation(operator="Op", kwargs={"default": None}) + result = format_value(op) + expected = """Op( + default=None, +)""" + assert result == expected + + +def test_real_world_example(): + """Test a realistic neural network architecture.""" + # Build a realistic example similar to architecture_search.py + conv1 = Operation( + operator="Conv2d", + kwargs={"in_channels": 3, "out_channels": 64, "kernel_size": [3, 4]}, + ) + relu1 = Operation(operator="ReLU") + pool1 = Operation(operator="MaxPool2d", kwargs={"kernel_size": 2, "stride": 2}) + + conv2 = Operation( + operator="Conv2d", + kwargs={"in_channels": 64, "out_channels": 128, "kernel_size": [3, 4]}, + ) + relu2 = Operation(operator="ReLU") + pool2 = Operation(operator="MaxPool2d", kwargs={"kernel_size": 2, "stride": 2}) + + flatten = Operation(operator="Flatten") + fc = Operation( + operator="Linear", kwargs={"in_features": 128 * 7 * 7, "out_features": 10} + ) + + model = Operation( + operator="Sequential", + args=([conv1, relu1, pool1, conv2, relu2, pool2, flatten, fc],), + ) + + result = format_value(model) + + # Verify key elements are present + assert "Sequential(" in result + assert "Conv2d(" in result + assert "in_channels=3" in result + assert "out_channels=64" in result + assert "kernel_size=[3, 4]" in result + assert "ReLU()," in result + assert "MaxPool2d(" in result + assert "Flatten()," in result + assert "Linear(" in result + assert "in_features=" in result + assert "out_features=10" in result + + +def test_categorical_with_operations(): + """Test formatting when a Categorical contains Operations - always expanded.""" + + class TestSpace(neps.PipelineSpace): + choice = neps.Categorical( + [ + Operation(operator="Conv2d", kwargs={"in_channels": 3, "kernel_size": 3}), + Operation(operator="ReLU"), + ] + ) + + # Sample and resolve + space = TestSpace() + resolved, _ = neps.space.neps_spaces.neps_space.resolve(space) + + # The resolved choice should be an Operation + assert isinstance(resolved.choice, Operation) + + # Should format properly - check for essential content + result = format_value(resolved.choice) + # Either Conv2d with both params, or ReLU + is_conv = ( + "Conv2d" in result and "in_channels=3" in result and "kernel_size=3" in result + ) + is_relu = result == "ReLU()" + assert is_conv or is_relu + + +def test_categorical_with_primitives(): + """Test formatting when a Categorical contains primitives.""" + + class TestSpace(neps.PipelineSpace): + choice = neps.Categorical(["adam", "sgd", "rmsprop"]) + + space = TestSpace() + resolved, _ = neps.space.neps_spaces.neps_space.resolve(space) + + # The resolved choice should be a string + assert isinstance(resolved.choice, str) + + # Should format as a simple string (identifiers don't get quotes) + result = format_value(resolved.choice) + assert result in ["adam", "sgd", "rmsprop"] + + +def test_categorical_with_mixed_types(): + """Test formatting when a Categorical contains mixed types.""" + + class TestSpace(neps.PipelineSpace): + choice = neps.Categorical( + [ + Operation(operator="Linear", kwargs={"in_features": 10}), + "simple_string", + 42, + ] + ) + + space = TestSpace() + resolved, _ = neps.space.neps_spaces.neps_space.resolve(space) + + # Should format appropriately based on what was chosen + result = format_value(resolved.choice) + + # Check it's one of the expected formats (identifiers don't get quotes) + possible_results = [ + "Linear(\n in_features=10,\n)", # Expanded format (3-space indent) + "Linear(in_features=10)", # Compact format (simple operation) + "simple_string", # Identifiers don't get quotes + "42", + ] + assert result in possible_results + + +def test_resolved_float_parameter(): + """Test formatting a resolved Float parameter.""" + + class TestSpace(neps.PipelineSpace): + lr = neps.Float(0.001, 0.1) + + space = TestSpace() + resolved, _ = neps.space.neps_spaces.neps_space.resolve(space) + + # Resolved Float becomes a float value + assert isinstance(resolved.lr, float) + + # Should format as a simple number + result = format_value(resolved.lr) + assert result == repr(resolved.lr) + + +def test_resolved_integer_parameter(): + """Test formatting a resolved Integer parameter.""" + + class TestSpace(neps.PipelineSpace): + batch_size = neps.Integer(16, 128) + + space = TestSpace() + resolved, _ = neps.space.neps_spaces.neps_space.resolve(space) + + # Resolved Integer becomes an int value + assert isinstance(resolved.batch_size, int) + + # Should format as a simple number + result = format_value(resolved.batch_size) + assert result == repr(resolved.batch_size) + + +if __name__ == "__main__": + # Run a quick test to see output + conv = Operation( + operator="Conv2d", + kwargs={"in_channels": 3, "out_channels": 64, "kernel_size": [3, 3]}, + ) + relu = Operation(operator="ReLU") + seq = Operation(operator="Sequential", args=([conv, relu],), kwargs={"dropout": 0.5}) + + import pytest + + pytest.main([__file__, "-v"]) diff --git a/tests/test_runtime/test_default_report_values.py b/tests/test_runtime/test_default_report_values.py index 8dec1bbbf..39fd216a1 100644 --- a/tests/test_runtime/test_default_report_values.py +++ b/tests/test_runtime/test_default_report_values.py @@ -7,7 +7,7 @@ from neps.optimizers import OptimizerInfo from neps.optimizers.algorithms import random_search from neps.runtime import DefaultWorker -from neps.space import Float, SearchSpace +from neps.space.neps_spaces.parameters import Float, PipelineSpace from neps.state import ( DefaultReportValues, NePSState, @@ -21,19 +21,26 @@ @fixture def neps_state(tmp_path: Path) -> NePSState: + class TestSpace(PipelineSpace): + a = Float(0, 1) + return NePSState.create_or_load( path=tmp_path / "neps_state", optimizer_info=OptimizerInfo(name="blah", info={"nothing": "here"}), optimizer_state=OptimizationState( budget=None, seed_snapshot=SeedSnapshot.new_capture(), shared_state={} ), + pipeline_space=TestSpace(), ) def test_default_values_on_error( neps_state: NePSState, ) -> None: - optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(pipeline_space=TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues( @@ -83,7 +90,10 @@ def eval_function(*args, **kwargs) -> float: def test_default_values_on_not_specified( neps_state: NePSState, ) -> None: - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues( @@ -131,7 +141,10 @@ def eval_function(*args, **kwargs) -> float: def test_default_value_objective_to_minimize_curve_take_objective_to_minimize_value( neps_state: NePSState, ) -> None: - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues( diff --git a/tests/test_runtime/test_error_handling_strategies.py b/tests/test_runtime/test_error_handling_strategies.py index b56c22666..25730c35a 100644 --- a/tests/test_runtime/test_error_handling_strategies.py +++ b/tests/test_runtime/test_error_handling_strategies.py @@ -11,7 +11,7 @@ from neps.optimizers import OptimizerInfo from neps.optimizers.algorithms import random_search from neps.runtime import DefaultWorker -from neps.space import Float, SearchSpace +from neps.space.neps_spaces.parameters import Float, PipelineSpace from neps.state import ( DefaultReportValues, NePSState, @@ -25,6 +25,9 @@ @fixture def neps_state(tmp_path: Path) -> NePSState: + class TestSpace(PipelineSpace): + a = Float(0, 1) + return NePSState.create_or_load( path=tmp_path / "neps_state", optimizer_info=OptimizerInfo(name="blah", info={"nothing": "here"}), @@ -33,6 +36,7 @@ def neps_state(tmp_path: Path) -> NePSState: seed_snapshot=SeedSnapshot.new_capture(), shared_state=None, ), + pipeline_space=TestSpace(), ) @@ -44,7 +48,10 @@ def test_worker_raises_when_error_in_self( neps_state: NePSState, on_error: OnErrorPossibilities, ) -> None: - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(TestSpace()) settings = WorkerSettings( on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), @@ -82,7 +89,10 @@ def eval_function(*args, **kwargs) -> float: def test_worker_raises_when_error_in_other_worker(neps_state: NePSState) -> None: - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.RAISE_ANY_ERROR, # <- Highlight default_report_values=DefaultReportValues(), @@ -140,7 +150,10 @@ def test_worker_does_not_raise_when_error_in_other_worker( neps_state: NePSState, on_error: OnErrorPossibilities, ) -> None: - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(TestSpace()) settings = WorkerSettings( on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), diff --git a/tests/test_runtime/test_save_evaluation_results.py b/tests/test_runtime/test_save_evaluation_results.py index e499e1afc..91b37584d 100644 --- a/tests/test_runtime/test_save_evaluation_results.py +++ b/tests/test_runtime/test_save_evaluation_results.py @@ -4,11 +4,10 @@ from pytest_cases import fixture -from neps import save_pipeline_results +from neps import Float, PipelineSpace, save_pipeline_results from neps.optimizers import OptimizerInfo from neps.optimizers.algorithms import random_search from neps.runtime import DefaultWorker -from neps.space import Float, SearchSpace from neps.state import ( DefaultReportValues, NePSState, @@ -28,11 +27,16 @@ def neps_state(tmp_path: Path) -> NePSState: optimizer_state=OptimizationState( budget=None, seed_snapshot=SeedSnapshot.new_capture(), shared_state={} ), + pipeline_space=ASpace(), ) +class ASpace(PipelineSpace): + a = Float(0, 1) + + def test_async_happy_path_changes_state(neps_state: NePSState) -> None: - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + optimizer = random_search(ASpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues( diff --git a/tests/test_runtime/test_stopping_criterion.py b/tests/test_runtime/test_stopping_criterion.py index 4884976c2..f26b25bbe 100644 --- a/tests/test_runtime/test_stopping_criterion.py +++ b/tests/test_runtime/test_stopping_criterion.py @@ -8,7 +8,8 @@ from neps.optimizers.algorithms import asha, random_search from neps.optimizers.optimizer import OptimizerInfo from neps.runtime import DefaultWorker -from neps.space import Float, Integer, SearchSpace +from neps.space import HPOFloat, HPOInteger, SearchSpace +from neps.space.neps_spaces.parameters import Float, PipelineSpace from neps.state import ( DefaultReportValues, NePSState, @@ -22,6 +23,9 @@ @fixture def neps_state(tmp_path: Path) -> NePSState: + class TestSpace(PipelineSpace): + a = Float(0, 1) + return NePSState.create_or_load( path=tmp_path / "neps_state", optimizer_info=OptimizerInfo(name="blah", info={"nothing": "here"}), @@ -30,13 +34,17 @@ def neps_state(tmp_path: Path) -> NePSState: seed_snapshot=SeedSnapshot.new_capture(), shared_state=None, ), + pipeline_space=TestSpace(), ) def test_evaluations_to_spend_stopping_criterion( neps_state: NePSState, ) -> None: - optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(pipeline_space=TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -104,7 +112,10 @@ def eval_function(*args, **kwargs) -> float: def test_multiple_criteria_set( neps_state: NePSState, ) -> None: - optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(pipeline_space=TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -172,7 +183,10 @@ def eval_function(*args, **kwargs) -> dict: def test_include_in_progress_evaluations_towards_maximum_with_work_eval_count( neps_state: NePSState, ) -> None: - optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(pipeline_space=TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -235,7 +249,10 @@ def eval_function(*args, **kwargs) -> float: def test_worker_wallclock_time(neps_state: NePSState) -> None: - optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(pipeline_space=TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -273,7 +290,10 @@ def eval_function(*args, **kwargs) -> float: def test_max_worker_evaluation_time(neps_state: NePSState) -> None: - optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer = random_search(pipeline_space=TestSpace()) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -326,7 +346,9 @@ def eval_function(*args, **kwargs) -> float: def test_fidelity_to_spend(neps_state: NePSState) -> None: optimizer = asha( - space=SearchSpace({"a": Float(0, 1), "b": Integer(2, 10, is_fidelity=True)}) + pipeline_space=SearchSpace( + {"a": HPOFloat(0, 1), "b": HPOInteger(2, 10, is_fidelity=True)} + ) ) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, diff --git a/tests/test_runtime/test_trajectory_and_metrics.py b/tests/test_runtime/test_trajectory_and_metrics.py new file mode 100644 index 000000000..05ee06312 --- /dev/null +++ b/tests/test_runtime/test_trajectory_and_metrics.py @@ -0,0 +1,803 @@ +"""Tests for extended trajectory and metrics functionality in NePS.""" + +from __future__ import annotations + +import re +import tempfile +import time +from pathlib import Path + +import pytest +from filelock import FileLock + +import neps +from neps.optimizers import algorithms +from neps.runtime import DefaultWorker +from neps.space.neps_spaces.parameters import ( + Float, + Integer, + IntegerFidelity, + PipelineSpace, +) +from neps.state.neps_state import NePSState +from neps.state.optimizer import OptimizationState +from neps.state.seed_snapshot import SeedSnapshot +from neps.state.settings import ( + DefaultReportValues, + OnErrorPossibilities, + WorkerSettings, +) +from neps.state.trial import ( + MetaData, + State as TrialState, + Trial, +) + + +class SimpleSpace(PipelineSpace): + """Simple space for testing metrics functionality.""" + + x = Float(lower=0.0, upper=1.0) + y = Integer(lower=1, upper=10) + + +class SpaceWithFidelity(PipelineSpace): + """Space with fidelity for testing multi-fidelity metrics.""" + + x = Float(lower=0.0, upper=1.0) + y = Integer(lower=1, upper=10) + epochs = IntegerFidelity(lower=1, upper=50) + + +def simple_evaluation(x: float, y: int) -> dict: + """Simple evaluation function that returns multiple metrics.""" + return { + "objective_to_minimize": x + y, + "accuracy": max(0.0, 1.0 - (x + y) / 11.0), # Dummy accuracy metric + "training_time": x * 10 + y, # Dummy training time + "memory_usage": y * 100, # Dummy memory usage + "custom_metric": x * y, # Custom metric + } + + +def fidelity_evaluation(x: float, y: int, epochs: int) -> dict: + """Evaluation function with fidelity that affects metrics.""" + base_objective = x + y + fidelity_factor = epochs / 50.0 # Scale based on fidelity + + return { + "objective_to_minimize": ( + base_objective / fidelity_factor + ), # Better with more epochs + "accuracy": min(1.0, fidelity_factor * (1.0 - base_objective / 11.0)), + "training_time": epochs * (x * 10 + y), # More epochs = more time + "memory_usage": y * 100 + epochs * 10, # Memory increases with epochs + "convergence_rate": 1.0 / epochs, # Faster convergence with more epochs + "epochs_used": epochs, # Track actual epochs used + } + + +def failing_evaluation(x: float, y: int) -> dict: + """Evaluation that sometimes fails to test error handling.""" + if x > 0.8 or y > 8: + raise ValueError("Simulated failure for testing") + + return { + "objective_to_minimize": x + y, + "success_rate": 1.0, + } + + +# ===== Test basic trajectory and metrics ===== + + +def test_basic_trajectory_functionality(): + """Test basic trajectory functionality without checking specific file structure.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "basic_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Check that some optimization files were created + assert root_directory.exists() + + # Find the summary directory and check for result files + summary_dir = root_directory / "summary" + assert summary_dir.exists(), "Summary directory should exist" + + # Check for best config file + best_config_file = summary_dir / "best_config.txt" + assert best_config_file.exists(), "Best config file should exist" + + # Check if trajectory file contains our evaluation results + best_config_content = best_config_file.read_text() + assert "Objective to minimize" in best_config_content # Different casing + + # Check for CSV files that contain the optimization summary + csv_files = list(summary_dir.glob("*.csv")) + assert len(csv_files) > 0, "Should have CSV summary files" + + # Check that basic optimization data is present + csv_content = csv_files[0].read_text() + assert "objective_to_minimize" in csv_content, "Should contain objective values" + + +def test_best_config_with_multiple_metrics(): + """Test that best_config file contains multiple metrics.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "best_config_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=5, + overwrite_root_directory=True, + ) + + # Check that best_config file exists + best_config_file = root_directory / "summary" / "best_config.txt" + assert best_config_file.exists(), "Best config file should exist" + + # Read and verify best config contains multiple metrics + best_config_content = best_config_file.read_text() + + # Should contain the primary objective + assert "Objective to minimize" in best_config_content + + # Note: Additional metrics may not be persisted to summary files + # They are used during evaluation but only the main objective is saved + # Should contain configuration parameters + assert ( + "x" in best_config_content or "SAMPLING__Resolvable.x" in best_config_content + ) + assert ( + "y" in best_config_content or "SAMPLING__Resolvable.y" in best_config_content + ) + + +def test_trajectory_with_fidelity(): + """Test trajectory with fidelity-based evaluation.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "fidelity_test" + + # Run optimization with fidelity + neps.run( + evaluate_pipeline=fidelity_evaluation, + pipeline_space=SpaceWithFidelity(), + optimizer=("neps_random_search", {"ignore_fidelity": True}), + root_directory=str(root_directory), + evaluations_to_spend=10, + overwrite_root_directory=True, + ) + + # Check trajectory file + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + assert trajectory_file.exists() + + trajectory_content = trajectory_file.read_text() + + # Should contain basic optimization data + assert "Config ID" in trajectory_content + assert "Objective" in trajectory_content + + # Should track configuration parameters (including fidelity if preserved) + assert any( + param in trajectory_content + for param in ["x", "y", "epochs", "SAMPLING__Resolvable"] + ) + + +def test_cumulative_metrics_tracking(): + """Test that cumulative evaluations are tracked in trajectory files.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "cumulative_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=5, + overwrite_root_directory=True, + ) + + # Read trajectory + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + trajectory_content = trajectory_file.read_text() + + # Should have the expected header + assert ( + "Best configs and their objectives across evaluations:" in trajectory_content + ) + + # Should track cumulative evaluations + assert "Cumulative evaluations:" in trajectory_content + + # Should have multiple config entries (at least some evaluations) + config_count = trajectory_content.count("Config ID:") + assert config_count >= 1, "Should have at least one config entry" + + # Should have objective values + assert "Objective to minimize:" in trajectory_content + + +# ===== Test error handling in metrics ===== + + +def test_trajectory_with_failed_evaluations(): + """Test that trajectory handles failed evaluations correctly.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "error_test" + + # Run optimization that will have some failures + neps.run( + evaluate_pipeline=failing_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=15, # More evaluations to ensure some failures + overwrite_root_directory=True, + ignore_errors=True, # Allow continuing after errors + ) + + # Check that trajectory file exists + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + assert trajectory_file.exists() + + # Read trajectory + trajectory_content = trajectory_file.read_text() + lines = trajectory_content.strip().split("\n") + + # Should have at least some successful evaluations + assert len(lines) >= 2 # Header + at least one evaluation + + # Check that errors are handled gracefully + # (The exact behavior may vary, but the file should exist and be readable) + assert "Objective to minimize" in trajectory_content # Different casing + + +# ===== Test hyperband-specific metrics ===== + + +def test_neps_hyperband_metrics(): + """Test that neps_hyperband produces extended metrics.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "hyperband_test" + + # Run neps_hyperband optimization + neps.run( + evaluate_pipeline=fidelity_evaluation, + pipeline_space=SpaceWithFidelity(), + optimizer=algorithms.neps_hyperband, + root_directory=str(root_directory), + fidelities_to_spend=20, # Use fidelities_to_spend for mf optimizers + overwrite_root_directory=True, + ) + + # Check trajectory file + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + assert trajectory_file.exists() + + trajectory_content = trajectory_file.read_text() + + # Should contain basic optimization data + assert "Objective" in trajectory_content + + # Should contain configuration information + assert any( + param in trajectory_content for param in ["epochs", "SAMPLING__Resolvable"] + ) + + # Should have multiple evaluations with different fidelities + lines = trajectory_content.strip().split("\n") + assert len(lines) >= 5 # Should have some evaluations + + +# ===== Test metrics with different optimizers ===== + + +@pytest.mark.parametrize( + "optimizer", + [ + algorithms.neps_random_search, + algorithms.complex_random_search, + ], +) +def test_metrics_with_different_optimizers(optimizer): + """Test that txt file format is consistent across different optimizers.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / f"optimizer_test_{optimizer.__name__}" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=optimizer, + root_directory=str(root_directory), + evaluations_to_spend=5, + overwrite_root_directory=True, + ) + + # Check files exist + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + best_config_file = root_directory / "summary" / "best_config.txt" + + assert trajectory_file.exists() + assert best_config_file.exists() + + # Check contents match expected txt format (only objective_to_minimize is tracked) + trajectory_content = trajectory_file.read_text() + best_config_content = best_config_file.read_text() + + # Both should contain the standard txt file format elements + for content in [trajectory_content, best_config_content]: + assert "Config ID:" in content + assert "Objective to minimize:" in content + assert "Cumulative evaluations:" in content + assert "Config:" in content + + # Trajectory file should have the header + assert ( + "Best configs and their objectives across evaluations:" in trajectory_content + ) + + +# ===== Test metric value validation ===== + + +def test_metric_values_are_reasonable(): + """Test that reported objective values are reasonable in txt files.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "validation_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=5, + overwrite_root_directory=True, + ) + + # Read trajectory and parse objective values + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + trajectory_content = trajectory_file.read_text() + + # Extract objective values from the actual txt format + objective_matches = re.findall( + r"Objective to minimize: ([\d.]+)", trajectory_content + ) + + # Check that we found some objectives + assert len(objective_matches) > 0, "No objective values found in trajectory" + + # Check each objective value is reasonable + for obj_str in objective_matches: + objective = float(obj_str) + # Objective should be in reasonable range (x+y where x in [0,1], y in [1,10]) + assert 1.0 <= objective <= 11.0, ( + f"Objective {objective} out of expected range [1.0, 11.0]" + ) + + +# ===== Test file format and structure ===== + + +def test_trajectory_file_format(): + """Test that trajectory txt file has correct format.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "format_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Check trajectory file format (txt format, not CSV) + trajectory_file = root_directory / "summary" / "best_config_trajectory.txt" + trajectory_content = trajectory_file.read_text() + + # Should have the expected txt file structure + assert ( + "Best configs and their objectives across evaluations:" in trajectory_content + ) + assert "Config ID:" in trajectory_content + assert "Objective to minimize:" in trajectory_content + assert "Cumulative evaluations:" in trajectory_content + assert "Config:" in trajectory_content + + # Should have separator lines + assert ( + "--------------------------------------------------------------------------------" + in trajectory_content + ) + + +def test_results_directory_structure(): + """Test that results directory has expected structure.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "structure_test" + + # Run optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, + ) + + # Check directory structure + results_dir = root_directory / "summary" + assert results_dir.exists() + assert results_dir.is_dir() + + # Check expected files + expected_files = ["best_config_trajectory.txt", "best_config.txt"] + for filename in expected_files: + file_path = results_dir / filename + assert file_path.exists(), f"Expected file {filename} should exist" + assert file_path.is_file(), f"{filename} should be a file" + + # File should not be empty + content = file_path.read_text() + assert len(content.strip()) > 0, f"{filename} should not be empty" + + +def test_neps_revisit_run_with_trajectory(): + """Test that NePS can revisit an earlier run and use incumbent trajectory.""" + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "revisit_test" + + # First run - create initial optimization + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=3, + overwrite_root_directory=True, # Start fresh + ) + + # Check that initial files were created + summary_dir = root_directory / "summary" + assert summary_dir.exists() + best_config_file = summary_dir / "best_config.txt" + trajectory_file = summary_dir / "best_config_trajectory.txt" + assert best_config_file.exists() + assert trajectory_file.exists() + + # Read initial trajectory + initial_trajectory = trajectory_file.read_text() + assert "Config ID:" in initial_trajectory + assert "Objective to minimize:" in initial_trajectory + + # Second run - revisit without overwriting + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=2, # Add 2 more evaluations + overwrite_root_directory=False, # Don't overwrite, continue from previous + ) + + # Check that trajectory was updated with new evaluations + updated_trajectory = trajectory_file.read_text() + + # The updated trajectory should contain the original entries plus new ones + assert len(updated_trajectory) >= len(initial_trajectory) + assert "Config ID:" in updated_trajectory + assert "Objective to minimize:" in updated_trajectory + + # Should have evidence of multiple evaluations + # Note: trajectory.txt only tracks BEST configs, not all evaluations + # So we check that the files still have the expected format and content + assert "Config ID:" in updated_trajectory + assert "Objective to minimize:" in updated_trajectory + + # The updated content should be at least as long (potentially with timing info + # added) + assert len(updated_trajectory) >= len(initial_trajectory), ( + "Updated trajectory should have at least the same content" + ) + + +@pytest.mark.parametrize("run_number", range(10)) +def test_continue_finished_run_with_higher_budget(run_number): + """Test continuing a finished run with overwrite_root_directory=False. + + This test runs 10 times with different random seeds to catch any + non-deterministic bugs in state management. + + Uses a HIGH initial budget to likely find a very good solution, then continues + with a SMALL additional budget. This makes bugs more obvious because: + - New evaluations will likely be worse than the initial best + - If bugs exist, they'll cause the worse configs to incorrectly appear as "best" + - Tests that the incumbent is properly preserved across continuation + + Verifies: + 1. Best config file contains the actual best configuration from all evaluations + 2. Cumulative metrics in best_config.txt are correct + 3. The best found improves or stays the same compared to initial run + 4. Trajectory objectives are monotonically non-increasing + """ + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / f"continue_test_run_{run_number}" + + # First run - complete optimization with HIGH budget to find a good solution + initial_budget = 20 + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=initial_budget, + overwrite_root_directory=True, + ) # Verify first run completed + summary_dir = root_directory / "summary" + best_config_file = summary_dir / "best_config.txt" + trajectory_file = summary_dir / "best_config_trajectory.txt" + + assert best_config_file.exists() + assert trajectory_file.exists() + + # Read initial results + initial_best_config = best_config_file.read_text() + trajectory_file.read_text() + + # Extract initial best objective + initial_obj_match = re.search( + r"Objective to minimize: ([\d.]+)", initial_best_config + ) + assert initial_obj_match is not None + initial_best_objective = float(initial_obj_match.group(1)) + + # Extract initial cumulative evaluations + initial_cum_match = re.search( + r"Cumulative evaluations: (\d+)", initial_best_config + ) + assert initial_cum_match is not None + initial_cumulative = int(initial_cum_match.group(1)) + + # Second run - continue with SMALL additional budget + # With the high initial budget, we likely found a good solution + # New evaluations will probably be worse, exposing bugs if they exist + additional_budget = 5 + total_expected_budget = initial_budget + additional_budget + + neps.run( + evaluate_pipeline=simple_evaluation, + pipeline_space=SimpleSpace(), + optimizer=algorithms.neps_random_search, + root_directory=str(root_directory), + evaluations_to_spend=additional_budget, + overwrite_root_directory=False, # Continue from previous run + ) + + # Read final results + final_best_config = best_config_file.read_text() + final_trajectory = trajectory_file.read_text() + + # ===== Test 1: Verify best_config.txt contains a valid best configuration ===== + final_obj_match = re.search(r"Objective to minimize: ([\d.]+)", final_best_config) + assert final_obj_match is not None + final_best_objective = float(final_obj_match.group(1)) + + # The final best should be <= initial best (can only improve or stay same) + assert final_best_objective <= initial_best_objective, ( + "Best objective regressed when continuing run: " + f"initial={initial_best_objective}, final={final_best_objective}" + ) + + # ===== Test 2: Verify cumulative metrics are correct ===== + # Extract cumulative evaluations from best_config.txt + final_cum_match = re.search(r"Cumulative evaluations: (\d+)", final_best_config) + assert final_cum_match is not None, "Should have cumulative evaluations" + final_cumulative_evals = int(final_cum_match.group(1)) + + # The cumulative evaluations should be >= initial cumulative + # (we added more evaluations in the second run) + assert final_cumulative_evals >= initial_cumulative, ( + f"Final cumulative evaluations {final_cumulative_evals} should be >= " + f"initial cumulative {initial_cumulative}" + ) + + # The cumulative evaluations should be <= total budget + # (may be less if some evaluations failed or optimizer stopped early) + assert final_cumulative_evals <= total_expected_budget, ( + f"Cumulative evaluations {final_cumulative_evals} should not exceed " + f"total budget {total_expected_budget}" + ) + + # ===== Test 3: Verify trajectory contains the improvement history ===== + trajectory_objectives = re.findall( + r"Objective to minimize: ([\d.]+)", final_trajectory + ) + assert len(trajectory_objectives) > 0, "Should have trajectory objectives" + + # Convert to floats + objectives = [float(obj) for obj in trajectory_objectives] + + # The minimum objective in trajectory should be the true best across all runs + min_trajectory_objective = min(objectives) + + # The best_config.txt should show the minimum objective from the trajectory + assert abs(final_best_objective - min_trajectory_objective) < 1e-10, ( + f"best_config.txt shows {final_best_objective} but " + f"trajectory minimum is {min_trajectory_objective}" + ) + + # ===== Test 4: Verify trajectory is monotonically non-increasing ===== + # The trajectory should be monotonically non-increasing (each entry should be + # better than or equal to the previous) + is_monotonic = all( + objectives[i] <= objectives[i - 1] for i in range(1, len(objectives)) + ) + + assert is_monotonic, ( + f"Trajectory is not monotonically non-increasing: {objectives}" + ) + + # ===== Test 5: Verify the config in best_config.txt is valid ===== + # The format is "Config: {dict}" so we look for that pattern + assert "Config:" in final_best_config, "Should have config section" + + # Should contain parameter values (x and y, possibly with SAMPLING__ prefix) + assert any( + param in final_best_config + for param in ["x", "y", "'x'", "'y'", "SAMPLING__Resolvable"] + ), "Config should contain parameter values" + + # ===== Test 6: Verify the objective value is in the valid range ===== + assert 1.0 <= final_best_objective <= 11.0, ( + f"Best objective {final_best_objective} should be in range [1.0, 11.0]" + ) + + # ===== Test 7: Verify trajectory file format is correct ===== + assert ( + "Best configs and their objectives across evaluations:" in final_trajectory + ), "Should have trajectory header" + + assert "Config ID:" in final_trajectory, "Should have config IDs" + assert ( + "--------------------------------------------------------------------------------" + in final_trajectory + ), "Should have separator lines" + + # ===== Test 8: Verify we actually did more evaluations ===== + # Count config entries in trajectory (though trajectory only shows improvements) + config_count = final_trajectory.count("Config ID:") + assert config_count >= 1, "Should have at least one config entry" + + +def test_best_config_multiobjective_frontier(): + """Test that best_config.txt for a multi-objective run contains both + Pareto-optimal configurations (two entries) when two non-dominated + objective vectors are present. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + root_directory = Path(tmp_dir) / "mo_test" + + # Create an empty NePS state (optimizer info is minimal) + optimizer_info = {"name": "test_opt", "info": {}} + state = NePSState.create_or_load( + path=root_directory, + optimizer_info=optimizer_info, + optimizer_state=OptimizationState( + budget=None, + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + ) + + # Build two completed trials with multi-objective vectors (non-dominated) + t1_meta = MetaData( + id="t1", + location=str(state._trial_repo.directory / "config_t1"), + state=TrialState.SUCCESS, + previous_trial_id=None, + previous_trial_location=None, + sampling_worker_id="external", + time_sampled=1.0, + ) + t1 = Trial(config={"which": 0}, metadata=t1_meta, report=None) + r1 = t1.set_complete( + report_as="success", + time_end=time.time(), + objective_to_minimize=[1.0, 0], + cost=None, + learning_curve=None, + err=None, + tb=None, + extra={}, + evaluation_duration=0.0, + ) + t1.report = r1 + + t2_meta = MetaData( + id="t2", + location=str(state._trial_repo.directory / "config_t2"), + state=TrialState.SUCCESS, + previous_trial_id=None, + previous_trial_location=None, + sampling_worker_id="external", + time_sampled=2.0, + ) + t2 = Trial(config={"which": 1}, metadata=t2_meta, report=None) + r2 = t2.set_complete( + report_as="success", + time_end=time.time(), + objective_to_minimize=[0, 1], + cost=None, + learning_curve=None, + err=None, + tb=None, + extra={}, + evaluation_duration=0.0, + ) + t2.report = r2 + + trials = {t1.id: t1, t2.id: t2} + + # Prepare summary paths and lock + summary_dir = root_directory / "summary" + summary_dir.mkdir(parents=True, exist_ok=True) + improvement_trace_path = summary_dir / "best_config_trajectory.txt" + best_config_path = summary_dir / "best_config.txt" + improvement_trace_path.touch() + best_config_path.touch() + + trace_lock = FileLock(str(root_directory / ".trace.lock")) + + # Create a minimal worker (optimizer/eval fn not used by load_incumbent_trace) + settings = WorkerSettings( + on_error=OnErrorPossibilities.IGNORE, + default_report_values=DefaultReportValues(), + batch_size=None, + evaluations_to_spend=None, + include_in_progress_evaluations_towards_maximum=False, + cost_to_spend=None, + fidelities_to_spend=None, + max_evaluation_time_total_seconds=None, + max_wallclock_time_seconds=None, + ) + + worker = DefaultWorker( + state=state, + settings=settings, + evaluation_fn=dict, + optimizer=lambda: None, + worker_id="w_external", + ) + + # Call the function that writes best_config for the given trials + worker.load_incumbent_trace( + trials, trace_lock, improvement_trace_path, best_config_path + ) + + content = best_config_path.read_text() + + # Should contain two Config ID entries for the two non-dominated solutions + assert content.count("Config ID:") == 2 + assert "t1" in content + assert "t2" in content diff --git a/tests/test_runtime/test_worker_creation.py b/tests/test_runtime/test_worker_creation.py index 372031e1d..fbef648ae 100644 --- a/tests/test_runtime/test_worker_creation.py +++ b/tests/test_runtime/test_worker_creation.py @@ -4,6 +4,7 @@ import pytest +from neps import Float, PipelineSpace from neps.optimizers import OptimizerInfo from neps.optimizers.algorithms import random_search from neps.runtime import ( @@ -12,7 +13,6 @@ OnErrorPossibilities, WorkerSettings, ) -from neps.space import Float, SearchSpace from neps.state import NePSState, OptimizationState, SeedSnapshot @@ -24,9 +24,14 @@ def neps_state(tmp_path: Path) -> NePSState: optimizer_state=OptimizationState( budget=None, seed_snapshot=SeedSnapshot.new_capture(), shared_state={} ), + pipeline_space=ASpace(), ) +class ASpace(PipelineSpace): + a = Float(0, 1) + + def test_create_worker_manual_id(neps_state: NePSState) -> None: settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, @@ -44,7 +49,8 @@ def eval_fn(config: dict) -> float: return 1.0 test_worker_id = "my_worker_123" - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + + optimizer = random_search(ASpace()) worker = DefaultWorker.new( state=neps_state, @@ -74,7 +80,7 @@ def test_create_worker_auto_id(neps_state: NePSState) -> None: def eval_fn(config: dict) -> float: return 1.0 - optimizer = random_search(SearchSpace({"a": Float(0, 1)})) + optimizer = random_search(ASpace()) worker = DefaultWorker.new( state=neps_state, diff --git a/tests/test_search_space.py b/tests/test_search_space.py index 73073a0cc..7182463fa 100644 --- a/tests/test_search_space.py +++ b/tests/test_search_space.py @@ -2,12 +2,17 @@ import pytest -from neps import Categorical, Constant, Float, Integer, SearchSpace +from neps.space import SearchSpace +from neps.space.parameters import HPOCategorical, HPOConstant, HPOFloat, HPOInteger def test_search_space_orders_parameters_by_name(): - unsorted = SearchSpace({"b": Float(0, 1), "c": Float(0, 1), "a": Float(0, 1)}) - expected = SearchSpace({"a": Float(0, 1), "b": Float(0, 1), "c": Float(0, 1)}) + unsorted = SearchSpace( + {"b": HPOFloat(0, 1), "c": HPOFloat(0, 1), "a": HPOFloat(0, 1)} + ) + expected = SearchSpace( + {"a": HPOFloat(0, 1), "b": HPOFloat(0, 1), "c": HPOFloat(0, 1)} + ) assert unsorted == expected @@ -15,17 +20,20 @@ def test_multipe_fidelities_raises_error(): # We should allow this at some point, but until we do, raise an error with pytest.raises(ValueError, match="neps only supports one fidelity parameter"): SearchSpace( - {"a": Float(0, 1, is_fidelity=True), "b": Float(0, 1, is_fidelity=True)} + { + "a": HPOFloat(0, 1, is_fidelity=True), + "b": HPOFloat(0, 1, is_fidelity=True), + } ) def test_sorting_of_parameters_into_subsets(): elements = { - "a": Float(0, 1), - "b": Integer(0, 10), - "c": Categorical(["a", "b", "c"]), - "d": Float(0, 1, is_fidelity=True), - "x": Constant("x"), + "a": HPOFloat(0, 1), + "b": HPOInteger(0, 10), + "c": HPOCategorical(["a", "b", "c"]), + "d": HPOFloat(0, 1, is_fidelity=True), + "x": HPOConstant("x"), } space = SearchSpace(elements) assert space.elements == elements diff --git a/tests/test_search_space_parsing.py b/tests/test_search_space_parsing.py index 4fd2ea226..b94eb9781 100644 --- a/tests/test_search_space_parsing.py +++ b/tests/test_search_space_parsing.py @@ -4,7 +4,14 @@ import pytest -from neps.space import Categorical, Constant, Float, Integer, Parameter, parsing +from neps.space import ( + HPOCategorical, + HPOConstant, + HPOFloat, + HPOInteger, + Parameter, + parsing, +) @pytest.mark.parametrize( @@ -12,27 +19,27 @@ [ ( (0, 1), - Integer(0, 1), + HPOInteger(0, 1), ), ( ("1e3", "1e5"), - Integer(1e3, 1e5), + HPOInteger(1e3, 1e5), ), ( ("1e-3", "1e-1"), - Float(1e-3, 1e-1), + HPOFloat(1e-3, 1e-1), ), ( (1e-5, 1e-1), - Float(1e-5, 1e-1), + HPOFloat(1e-5, 1e-1), ), ( {"type": "float", "lower": 0.00001, "upper": "1e-1", "log": True}, - Float(0.00001, 0.1, log=True), + HPOFloat(0.00001, 0.1, log=True), ), ( {"type": "int", "lower": 3, "upper": 30, "is_fidelity": True}, - Integer(3, 30, is_fidelity=True), + HPOInteger(3, 30, is_fidelity=True), ), ( { @@ -42,27 +49,27 @@ "log": True, "is_fidelity": False, }, - Integer(100, 30000, log=True, is_fidelity=False), + HPOInteger(100, 30000, log=True, is_fidelity=False), ), ( {"type": "float", "lower": "3.3e-5", "upper": "1.5E-1"}, - Float(3.3e-5, 1.5e-1), + HPOFloat(3.3e-5, 1.5e-1), ), ( {"type": "cat", "choices": [2, "sgd", "10e-3"]}, - Categorical([2, "sgd", 0.01]), + HPOCategorical([2, "sgd", 0.01]), ), ( 0.5, - Constant(0.5), + HPOConstant(0.5), ), ( "1e3", - Constant(1000), + HPOConstant(1000), ), ( {"type": "cat", "choices": ["adam", "sgd", "rmsprop"]}, - Categorical(["adam", "sgd", "rmsprop"]), + HPOCategorical(["adam", "sgd", "rmsprop"]), ), ( { @@ -72,7 +79,7 @@ "prior": 3.3e-2, "prior_confidence": "high", }, - Float(0.00001, 0.1, log=True, prior=3.3e-2, prior_confidence="high"), + HPOFloat(0.00001, 0.1, log=True, prior=3.3e-2, prior_confidence="high"), ), ], ) diff --git a/tests/test_state/test_filebased_neps_state.py b/tests/test_state/test_filebased_neps_state.py index 5572abb4d..8180e19f6 100644 --- a/tests/test_state/test_filebased_neps_state.py +++ b/tests/test_state/test_filebased_neps_state.py @@ -13,6 +13,7 @@ from neps.exceptions import NePSError, TrialNotFoundError from neps.optimizers import OptimizerInfo +from neps.space.neps_spaces.parameters import Float, PipelineSpace from neps.state.err_dump import ErrDump from neps.state.neps_state import NePSState from neps.state.optimizer import BudgetInfo, OptimizationState @@ -47,11 +48,15 @@ def test_create_with_new_filebased_neps_state( optimizer_info: OptimizerInfo, optimizer_state: OptimizationState, ) -> None: + class TestSpace(PipelineSpace): + a = Float(0, 1) + new_path = tmp_path / "neps_state" neps_state = NePSState.create_or_load( path=new_path, optimizer_info=optimizer_info, optimizer_state=optimizer_state, + pipeline_space=TestSpace(), ) assert neps_state.lock_and_get_optimizer_info() == optimizer_info assert neps_state.lock_and_get_optimizer_state() == optimizer_state @@ -70,11 +75,15 @@ def test_create_or_load_with_load_filebased_neps_state( optimizer_info: OptimizerInfo, optimizer_state: OptimizationState, ) -> None: + class TestSpace(PipelineSpace): + a = Float(0, 1) + new_path = tmp_path / "neps_state" neps_state = NePSState.create_or_load( path=new_path, optimizer_info=optimizer_info, optimizer_state=optimizer_state, + pipeline_space=TestSpace(), ) # NOTE: This isn't a defined way to do this but we should check @@ -89,6 +98,7 @@ def test_create_or_load_with_load_filebased_neps_state( path=new_path, optimizer_info=optimizer_info, optimizer_state=different_state, + pipeline_space=TestSpace(), ) assert neps_state == neps_state2 @@ -98,27 +108,66 @@ def test_load_on_existing_neps_state( optimizer_info: OptimizerInfo, optimizer_state: OptimizationState, ) -> None: + class TestSpace(PipelineSpace): + a = Float(0, 1) + + new_path = tmp_path / "neps_state" + neps_state = NePSState.create_or_load( + path=new_path, + optimizer_info=optimizer_info, + optimizer_state=optimizer_state, + pipeline_space=TestSpace(), + ) + + neps_state2 = NePSState.create_or_load(path=new_path, load_only=True) + assert neps_state == neps_state2 + + +def test_pipeline_space_written_and_reloaded(tmp_path: Path) -> None: + class TestSpace(PipelineSpace): + a = Float(0, 1) + + optimizer_info = OptimizerInfo(name="test", info={"a": "b"}) + optimizer_state = OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ) + new_path = tmp_path / "neps_state" neps_state = NePSState.create_or_load( path=new_path, optimizer_info=optimizer_info, optimizer_state=optimizer_state, + pipeline_space=TestSpace(), ) + # Load-only should return the same state neps_state2 = NePSState.create_or_load(path=new_path, load_only=True) assert neps_state == neps_state2 + # And their pipeline spaces must serialize to the same bytes + import pickle + + assert pickle.dumps(neps_state._pipeline_space) == pickle.dumps( + neps_state2._pipeline_space + ) + def test_new_or_load_on_existing_neps_state_with_different_optimizer_info( tmp_path: Path, optimizer_info: OptimizerInfo, optimizer_state: OptimizationState, ) -> None: + class TestSpace(PipelineSpace): + a = Float(0, 1) + new_path = tmp_path / "neps_state" NePSState.create_or_load( path=new_path, optimizer_info=optimizer_info, optimizer_state=optimizer_state, + pipeline_space=TestSpace(), ) with pytest.raises(NePSError): @@ -126,4 +175,5 @@ def test_new_or_load_on_existing_neps_state_with_different_optimizer_info( path=new_path, optimizer_info=OptimizerInfo(name="randomlll", info={"e": "f"}), optimizer_state=optimizer_state, + pipeline_space=TestSpace(), ) diff --git a/tests/test_state/test_neps_state.py b/tests/test_state/test_neps_state.py index bf7512fae..d3a13401b 100644 --- a/tests/test_state/test_neps_state.py +++ b/tests/test_state/test_neps_state.py @@ -12,6 +12,7 @@ import pytest from pytest_cases import case, fixture, parametrize, parametrize_with_cases +import neps from neps.optimizers import ( AskFunction, OptimizerInfo, @@ -19,64 +20,62 @@ load_optimizer, ) from neps.optimizers.ask_and_tell import AskAndTell -from neps.space import ( +from neps.space import SearchSpace +from neps.space.neps_spaces.parameters import ( Categorical, - Constant, Float, Integer, - SearchSpace, + IntegerFidelity, + PipelineSpace, ) from neps.state import BudgetInfo, NePSState, OptimizationState, SeedSnapshot +from neps.state.trial import Report @case -def case_search_space_no_fid() -> SearchSpace: - return SearchSpace( - { - "a": Float(0, 1), - "b": Categorical(["a", "b", "c"]), - "c": Constant("a"), - "d": Integer(0, 10), - } - ) +def case_search_space_no_fid() -> PipelineSpace: + class Space(PipelineSpace): + a = Float(0, 1) + b = Categorical(("a", "b", "c")) + c = "a" + d = Integer(0, 10) + + return Space() @case -def case_search_space_with_fid() -> SearchSpace: - return SearchSpace( - { - "a": Float(0, 1), - "b": Categorical(["a", "b", "c"]), - "c": Constant("a"), - "d": Integer(0, 10), - "e": Integer(1, 10, is_fidelity=True), - } - ) +def case_search_space_with_fid() -> PipelineSpace: + class SpaceFid(PipelineSpace): + a = Float(0, 1) + b = Categorical(("a", "b", "c")) + c = "a" + d = Integer(0, 10) + e = IntegerFidelity(1, 10) + + return SpaceFid() @case -def case_search_space_no_fid_with_prior() -> SearchSpace: - return SearchSpace( - { - "a": Float(0, 1, prior=0.5), - "b": Categorical(["a", "b", "c"], prior="a"), - "c": Constant("a"), - "d": Integer(0, 10, prior=5), - } - ) +def case_search_space_no_fid_with_prior() -> PipelineSpace: + class SpacePrior(PipelineSpace): + a = Float(0, 1, prior=0.5, prior_confidence="medium") + b = Categorical(("a", "b", "c"), prior=0, prior_confidence="medium") + c = "a" + d = Integer(0, 10, prior=5, prior_confidence="medium") + + return SpacePrior() @case -def case_search_space_fid_with_prior() -> SearchSpace: - return SearchSpace( - { - "a": Float(0, 1, prior=0.5), - "b": Categorical(["a", "b", "c"], prior="a"), - "c": Constant("a"), - "d": Integer(0, 10, prior=5), - "e": Integer(1, 10, is_fidelity=True), - } - ) +def case_search_space_fid_with_prior() -> PipelineSpace: + class SpaceFidPrior(PipelineSpace): + a = Float(0, 1, prior=0.5, prior_confidence="medium") + b = Categorical(("a", "b", "c"), prior=0, prior_confidence="medium") + c = "a" + d = Integer(0, 10, prior=5, prior_confidence="medium") + e = IntegerFidelity(1, 10) + + return SpaceFidPrior() # See issue #121 @@ -96,12 +95,19 @@ def case_search_space_fid_with_prior() -> SearchSpace: "async_hb", "ifbo", "priorband", + "moasha", + "mo_hyperband", + "neps_priorband", + "neps_hyperband", ] NO_DEFAULT_FIDELITY_SUPPORT = [ "random_search", "grid_search", "bayesian_optimization", "pibo", + "neps_random_search", + "complex_random_search", + "neps_regularized_evolution", ] NO_DEFAULT_PRIOR_SUPPORT = [ "grid_search", @@ -112,6 +118,11 @@ def case_search_space_fid_with_prior() -> SearchSpace: "hyperband", "async_hb", "random_search", + "moasha", + "mo_hyperband", + "neps_random_search", + "complex_random_search", + "neps_regularized_evolution", ] REQUIRES_PRIOR = [ "pibo", @@ -127,41 +138,64 @@ def case_search_space_fid_with_prior() -> SearchSpace: "primo", ] +REQUIRES_NEPS_SPACE = [ + "neps_priorband", + "neps_random_search", + "complex_random_search", + "neps_hyperband", + "neps_regularized_evolution", + "neps_local_and_incumbent", +] + +REQUIRES_ADDTIONAL_SETUP = ["neps_local_and_incumbent"] + @fixture @parametrize("key", list(PredefinedOptimizers.keys())) @parametrize_with_cases("search_space", cases=".", prefix="case_search_space") def optimizer_and_key_and_search_space( - key: str, search_space: SearchSpace -) -> tuple[AskFunction, str, SearchSpace]: + key: str, search_space: PipelineSpace +) -> tuple[AskFunction, str, PipelineSpace | SearchSpace]: if key in JUST_SKIP: pytest.xfail(f"{key} is not instantiable") - if key in NO_DEFAULT_PRIOR_SUPPORT and any( - parameter.prior is not None for parameter in search_space.searchables.values() - ): + if key in NO_DEFAULT_PRIOR_SUPPORT and search_space.has_priors(): pytest.xfail(f"{key} crashed with a prior") - if search_space.fidelity is not None and key in NO_DEFAULT_FIDELITY_SUPPORT: + if search_space.fidelity_attrs and key in NO_DEFAULT_FIDELITY_SUPPORT: pytest.xfail(f"{key} crashed with a fidelity") - if key in REQUIRES_FIDELITY and search_space.fidelity is None: + if key in REQUIRES_FIDELITY and not search_space.fidelity_attrs: pytest.xfail(f"{key} requires a fidelity parameter") - if key in REQUIRES_PRIOR and all( - parameter.prior is None for parameter in search_space.searchables.values() - ): + if key in REQUIRES_PRIOR and not search_space.has_priors(): pytest.xfail(f"{key} requires a prior") - if key in REQUIRES_FIDELITY_MO and search_space.fidelity is None: + if key in REQUIRES_FIDELITY_MO and not search_space.fidelity_attrs: pytest.xfail(f"Multi-objective optimizer {key} requires a fidelity parameter") if key in REQUIRES_MO_PRIOR: pytest.xfail("No tests defined for PriMO yet") + if key in REQUIRES_ADDTIONAL_SETUP: + pytest.xfail(f"{key} requires additional setup not implemented in this test") + kwargs: dict[str, Any] = {} opt, _ = load_optimizer((key, kwargs), search_space) # type: ignore - return opt, key, search_space + converted_space = ( + neps.space.neps_spaces.neps_space.convert_neps_to_classic_search_space( + search_space + ) + ) + return ( + opt, + key, + ( + converted_space + if converted_space and key not in REQUIRES_NEPS_SPACE + else search_space + ), + ) @parametrize("optimizer_info", [OptimizerInfo(name="blah", info={"a": "b"})]) @@ -173,6 +207,9 @@ def case_neps_state_filebased( optimizer_info: OptimizerInfo, shared_state: dict[str, Any], ) -> NePSState: + class TestSpace(PipelineSpace): + a = Float(0, 1) + new_path = tmp_path / "neps_state" return NePSState.create_or_load( path=new_path, @@ -182,17 +219,23 @@ def case_neps_state_filebased( seed_snapshot=SeedSnapshot.new_capture(), shared_state=shared_state, ), + pipeline_space=TestSpace(), ) @parametrize_with_cases("neps_state", cases=".", prefix="case_neps_state") def test_sample_trial( neps_state: NePSState, - optimizer_and_key_and_search_space: tuple[AskFunction, str, SearchSpace], + optimizer_and_key_and_search_space: tuple[ + AskFunction, str, PipelineSpace | SearchSpace + ], capsys, ) -> None: optimizer, key, search_space = optimizer_and_key_and_search_space + if key in REQUIRES_ADDTIONAL_SETUP: + pytest.xfail(f"{key} requires additional setup not implemented in this test") + assert neps_state.lock_and_read_trials() == {} assert neps_state.lock_and_get_next_pending_trial() is None assert neps_state.lock_and_get_next_pending_trial(n=10) == [] @@ -202,8 +245,24 @@ def test_sample_trial( for k, v in trial1.config.items(): assert v is not None, f"'{k}' is None in {trial1.config}" - for name in search_space: - assert name in trial1.config, f"'{name}' is not in {trial1.config}" + if isinstance(search_space, SearchSpace): + for name in search_space: + assert name in trial1.config, f"'{name}' is not in {trial1.config}" + else: + config = neps.space.neps_spaces.neps_space.NepsCompatConverter().from_neps_config( + trial1.config + ) + resolved_pipeline, _ = neps.space.neps_spaces.neps_space.resolve( + pipeline=search_space, + domain_sampler=neps.space.neps_spaces.neps_space.OnlyPredefinedValuesSampler( + predefined_samplings=config.predefined_samplings + ), + environment_values=config.environment_values, + ) + for name in search_space.get_attrs(): + assert name in resolved_pipeline.get_attrs(), ( + f"'{name}' is not in {resolved_pipeline.get_attrs()}" + ) # HACK: Unfortunatly due to windows, who's time.time() is not very # precise, we need to introduce a sleep -_- @@ -218,8 +277,24 @@ def test_sample_trial( for k, v in trial1.config.items(): assert v is not None, f"'{k}' is None in {trial1.config}" - for name in search_space: - assert name in trial1.config, f"'{name}' is not in {trial1.config}" + if isinstance(search_space, SearchSpace): + for name in search_space: + assert name in trial1.config, f"'{name}' is not in {trial1.config}" + else: + config = neps.space.neps_spaces.neps_space.NepsCompatConverter().from_neps_config( + trial1.config + ) + resolved_pipeline, _ = neps.space.neps_spaces.neps_space.resolve( + pipeline=search_space, + domain_sampler=neps.space.neps_spaces.neps_space.OnlyPredefinedValuesSampler( + predefined_samplings=config.predefined_samplings + ), + environment_values=config.environment_values, + ) + for name in search_space.get_attrs(): + assert name in resolved_pipeline.get_attrs(), ( + f"'{name}' is not in {resolved_pipeline.get_attrs()}" + ) assert trial1 != trial2 @@ -230,7 +305,9 @@ def test_sample_trial( def test_optimizers_work_roughly( - optimizer_and_key_and_search_space: tuple[AskFunction, str, SearchSpace], + optimizer_and_key_and_search_space: tuple[ + AskFunction, str, PipelineSpace | SearchSpace + ], ) -> None: opt, key, search_space = optimizer_and_key_and_search_space ask_and_tell = AskAndTell(opt) @@ -241,3 +318,64 @@ def test_optimizers_work_roughly( ask_and_tell.tell(trial, [1.0, 2.0]) else: ask_and_tell.tell(trial, 1.0) + + +@fixture +def neps_state(tmp_path: Path) -> NePSState: + class TestSpace(PipelineSpace): + a = Float(0, 1) + + return NePSState.create_or_load( + path=tmp_path / "neps_state", + optimizer_info=OptimizerInfo(name="random_search", info={}), + optimizer_state=OptimizationState( + budget=None, + seed_snapshot=SeedSnapshot.new_capture(), + shared_state=None, + ), + pipeline_space=TestSpace(), + ) + + +def test_get_valid_evaluated_trials( + neps_state: NePSState, +) -> None: + optimizer, _ = load_optimizer(("random_search", {}), neps_state._pipeline_space) + trial1 = neps_state.lock_and_sample_trial(optimizer=optimizer, worker_id="1") + trial2 = neps_state.lock_and_sample_trial(optimizer=optimizer, worker_id="1") + trial3 = neps_state.lock_and_sample_trial(optimizer=optimizer, worker_id="1") + + report1 = Report( + objective_to_minimize=0.5, + err=None, + cost=0, + learning_curve=[0], + extra={}, + tb=None, + reported_as="success", + evaluation_duration=1, + ) + neps_state.lock_and_report_trial_evaluation( + trial=trial1, + report=report1, + worker_id="1", + ) + + report2 = Report( + objective_to_minimize=float("nan"), + err=None, + cost=0, + learning_curve=[0], + extra={}, + tb=None, + reported_as="success", + evaluation_duration=1, + ) + neps_state.lock_and_report_trial_evaluation( + trial=trial2, report=report2, worker_id="1" + ) + + valid_trials = neps_state._trial_repo.get_valid_evaluated_trials() + assert len(valid_trials) == 1 + assert trial1.id in valid_trials + assert trial3.id not in valid_trials diff --git a/tests/test_state/test_search_space_persistence.py b/tests/test_state/test_search_space_persistence.py new file mode 100644 index 000000000..b322e14e7 --- /dev/null +++ b/tests/test_state/test_search_space_persistence.py @@ -0,0 +1,434 @@ +"""Tests for search space persistence in NePSState. + +This file focuses on low-level NePSState functionality: +- Saving and loading search spaces (PipelineSpace and SearchSpace) +- Backward compatibility (runs without search space) +- Testing utility functions like load_pipeline_space and load_optimizer_info + +For higher-level integration tests and validation logic, see +test_search_space_validation.py. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from neps.exceptions import NePSError +from neps.optimizers import OptimizerInfo +from neps.space import HPOCategorical, HPOFloat, HPOInteger, SearchSpace +from neps.space.neps_spaces.parameters import Categorical, Float, Integer, PipelineSpace +from neps.state import BudgetInfo, NePSState, OptimizationState, SeedSnapshot + + +class SimpleSpace(PipelineSpace): + """Simple test space with various parameter types.""" + + a = Float(0, 1) + b = Categorical(("x", "y", "z")) + c = Integer(0, 10) + + +class Space1(PipelineSpace): + """First test space for validation.""" + + x = Float(0, 10) + y = Integer(1, 10) + + +class Space2(PipelineSpace): + """Second test space (different from TestSpace1).""" + + x = Float(0, 10) + y = Integer(1, 20) # Different range + + +def test_search_space_saved_and_loaded_pipeline_space(tmp_path: Path) -> None: + """Test that PipelineSpace is saved and can be loaded back.""" + root_dir = tmp_path / "test_run" + pipeline_space = SimpleSpace() + + # Create state with search space + NePSState.create_or_load( + path=root_dir, + optimizer_info=OptimizerInfo(name="test", info={}), + optimizer_state=OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + pipeline_space=pipeline_space, + ) + + # Verify pipeline_space.pkl exists + assert (root_dir / "pipeline_space.pkl").exists() + + # Load state and verify search space + state2 = NePSState.create_or_load(path=root_dir, load_only=True) + loaded_space = state2.lock_and_get_search_space() + + assert loaded_space is not None + assert isinstance(loaded_space, PipelineSpace) + + # Verify the structure matches + assert "a" in loaded_space.get_attrs() + assert "b" in loaded_space.get_attrs() + assert "c" in loaded_space.get_attrs() + + +def test_search_space_saved_and_loaded_search_space(tmp_path: Path) -> None: + """Test that old-style SearchSpace is saved and can be loaded back.""" + root_dir = tmp_path / "test_run" + search_space = SearchSpace( + { + "a": HPOFloat(0, 1), + "b": HPOCategorical(["x", "y", "z"]), + "c": HPOInteger(0, 10), + } + ) + + # Create state with search space + NePSState.create_or_load( + path=root_dir, + optimizer_info=OptimizerInfo(name="test", info={}), + optimizer_state=OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + pipeline_space=search_space, + ) + + # Verify pipeline_space.pkl exists + assert (root_dir / "pipeline_space.pkl").exists() + + # Load state and verify search space + state2 = NePSState.create_or_load(path=root_dir, load_only=True) + loaded_space = state2.lock_and_get_search_space() + + assert loaded_space is not None + assert isinstance(loaded_space, SearchSpace) + assert "a" in loaded_space + assert "b" in loaded_space + assert "c" in loaded_space + + +def test_search_space_not_provided_backward_compatible(tmp_path: Path) -> None: + """Test that NePSState works without search space (backward compatibility).""" + root_dir = tmp_path / "test_run" + + # Create state WITHOUT search space + NePSState.create_or_load( + path=root_dir, + optimizer_info=OptimizerInfo(name="test", info={}), + optimizer_state=OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + ) + + # Verify pipeline_space.pkl does NOT exist + assert not (root_dir / "pipeline_space.pkl").exists() + + # Load state and verify search space is None + state2 = NePSState.create_or_load(path=root_dir, load_only=True) + loaded_space = state2.lock_and_get_search_space() + + assert loaded_space is None + + +def test_load_pipeline_space_function_pipeline_space(tmp_path: Path) -> None: + """Test the load_pipeline_space utility function with PipelineSpace.""" + from neps import load_pipeline_space + + root_dir = tmp_path / "test_run" + pipeline_space = SimpleSpace() + + # Create state with search space + NePSState.create_or_load( + path=root_dir, + optimizer_info=OptimizerInfo(name="test", info={}), + optimizer_state=OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + pipeline_space=pipeline_space, + ) + + # Load using the utility function + loaded_space = load_pipeline_space(root_dir) + + assert loaded_space is not None + assert isinstance(loaded_space, PipelineSpace) + assert "a" in loaded_space.get_attrs() + assert "b" in loaded_space.get_attrs() + assert "c" in loaded_space.get_attrs() + + +def test_load_pipeline_space_function_search_space(tmp_path: Path) -> None: + """Test the load_pipeline_space utility function with SearchSpace.""" + from neps import load_pipeline_space + + root_dir = tmp_path / "test_run" + search_space = SearchSpace( + { + "x": HPOFloat(0, 1), + "y": HPOInteger(1, 10), + } + ) + + # Create state with search space + NePSState.create_or_load( + path=root_dir, + optimizer_info=OptimizerInfo(name="test", info={}), + optimizer_state=OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + pipeline_space=search_space, + ) + + # Load using the utility function + loaded_space = load_pipeline_space(root_dir) + + assert loaded_space is not None + assert isinstance(loaded_space, SearchSpace) + assert "x" in loaded_space + assert "y" in loaded_space + + +def test_load_pipeline_space_function_not_found(tmp_path: Path) -> None: + """Test that load_pipeline_space raises FileNotFoundError for non-existent + directory. + """ + from neps import load_pipeline_space + + root_dir = tmp_path / "nonexistent" + + with pytest.raises(FileNotFoundError, match="No neps state found"): + load_pipeline_space(root_dir) + + +def test_load_pipeline_space_function_no_space_saved(tmp_path: Path) -> None: + """Test that load_pipeline_space raises ValueError when no search space was saved.""" + from neps import load_pipeline_space + + root_dir = tmp_path / "test_run" + + # Create state WITHOUT search space + NePSState.create_or_load( + path=root_dir, + optimizer_info=OptimizerInfo(name="test", info={}), + optimizer_state=OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + ) + + # Try to load - should raise ValueError + with pytest.raises(ValueError, match="No pipeline space was saved"): + load_pipeline_space(root_dir) + + +def test_load_optimizer_info_function(tmp_path: Path) -> None: + """Test the load_optimizer_info utility function.""" + from neps import load_optimizer_info + + root_dir = tmp_path / "test_run" + + # Create state with optimizer info + optimizer_info = OptimizerInfo( + name="bayesian_optimization", + info={"acquisition": "EI", "initial_design_size": 10}, + ) + NePSState.create_or_load( + path=root_dir, + optimizer_info=optimizer_info, + optimizer_state=OptimizationState( + budget=BudgetInfo(cost_to_spend=10, used_cost_budget=0), + seed_snapshot=SeedSnapshot.new_capture(), + shared_state={}, + ), + pipeline_space=SimpleSpace(), + ) + + # Load using the utility function + loaded_info = load_optimizer_info(root_dir) + + assert loaded_info["name"] == "bayesian_optimization" + assert loaded_info["info"]["acquisition"] == "EI" + assert loaded_info["info"]["initial_design_size"] == 10 + + +def test_load_optimizer_info_function_not_found(tmp_path: Path) -> None: + """Test that load_optimizer_info raises FileNotFoundError for non-existent + directory. + """ + from neps import load_optimizer_info + + root_dir = tmp_path / "nonexistent" + + with pytest.raises(FileNotFoundError, match="No neps state found"): + load_optimizer_info(root_dir) + + +def test_import_trials_saves_search_space(tmp_path: Path) -> None: + """Test that import_trials saves the search space to disk.""" + from neps import import_trials, load_pipeline_space + from neps.state.pipeline_eval import UserResultDict + + root_dir = tmp_path / "test_import" + + # Import trials with a search space + evaluated_trials = [ + ({"x": 1.0, "y": 5}, UserResultDict(objective_to_minimize=1.0)), + ({"x": 2.0, "y": 8}, UserResultDict(objective_to_minimize=2.0)), + ] + + import_trials( + evaluated_trials=evaluated_trials, + root_directory=root_dir, + pipeline_space=Space1(), + ) + + # Verify the search space was saved + loaded_space = load_pipeline_space(root_dir) + assert loaded_space is not None + # import_trials may convert PipelineSpace to SearchSpace, so check for SearchSpace + assert isinstance(loaded_space, Space1 | SearchSpace) + # Verify it has the correct parameters by checking the keys + if isinstance(loaded_space, SearchSpace): + assert "x" in loaded_space + assert "y" in loaded_space + + +def test_import_trials_validates_search_space(tmp_path: Path) -> None: + """Test that import_trials validates the search space against what's on disk.""" + from neps import import_trials + from neps.state.pipeline_eval import UserResultDict + + root_dir = tmp_path / "test_import_validate" + + # First import with one space + evaluated_trials = [ + ({"x": 1.0, "y": 5}, UserResultDict(objective_to_minimize=1.0)), + ] + + import_trials( + evaluated_trials=evaluated_trials, + root_directory=root_dir, + pipeline_space=Space1(), + ) + + # Try to import again with a different space - should raise error + with pytest.raises(NePSError, match="pipeline space on disk does not match"): + import_trials( + evaluated_trials=evaluated_trials, + root_directory=root_dir, + pipeline_space=Space2(), + ) + + +def test_import_trials_without_space_loads_from_disk(tmp_path: Path) -> None: + """Test that import_trials can load pipeline space from disk when not provided.""" + from neps import import_trials, load_pipeline_space + from neps.state.pipeline_eval import UserResultDict + + root_dir = tmp_path / "test_import_auto_load" + + # First import with explicit space + evaluated_trials_1 = [ + ({"x": 1.0, "y": 5}, UserResultDict(objective_to_minimize=1.0)), + ] + + import_trials( + evaluated_trials=evaluated_trials_1, + root_directory=root_dir, + pipeline_space=Space1(), + ) + + # Second import without providing space - should load from disk + evaluated_trials_2 = [ + ({"x": 2.0, "y": 8}, UserResultDict(objective_to_minimize=2.0)), + ] + + import_trials( + evaluated_trials=evaluated_trials_2, + root_directory=root_dir, + # pipeline_space not provided - should load from disk + ) + + # Verify both trials were imported successfully + loaded_space = load_pipeline_space(root_dir) + assert loaded_space is not None + + +def test_import_trials_without_space_fails_on_new_directory(tmp_path: Path) -> None: + """Test that import_trials raises error when space is not provided and directory + is new. + """ + from neps import import_trials + from neps.state.pipeline_eval import UserResultDict + + root_dir = tmp_path / "test_import_no_space_error" + + evaluated_trials = [ + ({"x": 1.0, "y": 5}, UserResultDict(objective_to_minimize=1.0)), + ] + + # Should raise error when no space provided and directory doesn't exist + with pytest.raises( + ValueError, match="pipeline_space is required when importing trials" + ): + import_trials( + evaluated_trials=evaluated_trials, + root_directory=root_dir, + # pipeline_space not provided + ) + + +def test_import_trials_validates_provided_space_against_disk(tmp_path: Path) -> None: + """Test that when both space is provided and exists on disk, they are validated.""" + from neps import import_trials + from neps.state.pipeline_eval import UserResultDict + + root_dir = tmp_path / "test_import_validation" + + # First import with TestSpace1 + evaluated_trials_1 = [ + ({"x": 1.0, "y": 5}, UserResultDict(objective_to_minimize=1.0)), + ] + + import_trials( + evaluated_trials=evaluated_trials_1, + root_directory=root_dir, + pipeline_space=Space1(), + ) + + # Second import explicitly providing the same space - should work + evaluated_trials_2 = [ + ({"x": 2.0, "y": 8}, UserResultDict(objective_to_minimize=2.0)), + ] + + import_trials( + evaluated_trials=evaluated_trials_2, + root_directory=root_dir, + pipeline_space=Space1(), + ) + + # Third import with different space - should fail + with pytest.raises(NePSError, match="pipeline space on disk does not match"): + import_trials( + evaluated_trials=evaluated_trials_2, + root_directory=root_dir, + pipeline_space=Space2(), + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_state/test_search_space_validation.py b/tests/test_state/test_search_space_validation.py new file mode 100644 index 000000000..0ad5e4cb3 --- /dev/null +++ b/tests/test_state/test_search_space_validation.py @@ -0,0 +1,252 @@ +"""Tests for search space validation and error handling. + +This file focuses on high-level integration tests through neps.run(): +- Strict validation (errors on mismatched search spaces) +- Auto-loading from disk +- Error handling when search space is missing +- Integration with load_config, status, and DDP runtime + +For low-level persistence tests, see test_search_space_persistence.py. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import pytest + +import neps +from neps.exceptions import NePSError +from neps.space import SearchSpace +from neps.space.neps_spaces.parameters import Float, Integer, PipelineSpace +from neps.state import NePSState + + +class Space1(PipelineSpace): + """First test pipeline space.""" + + x = Float(0.0, 1.0) + y = Integer(1, 10) + + +class Space2(PipelineSpace): + """Different test pipeline space.""" + + a = Float(0.0, 2.0) + b = Integer(5, 20) + + +def eval_fn1(**config): + """Evaluation function for TestSpace1.""" + return config["x"] + config["y"] + + +def eval_fn2(**config): + """Evaluation function for TestSpace2.""" + return config["a"] + config["b"] + + +def test_error_on_mismatched_search_space(tmp_path: Path): + """Test that providing a different search space raises an error (strict + validation). + """ + root_dir = tmp_path / "test_error" + + # Create initial state with TestSpace1 + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Try to continue with TestSpace2 - should raise NePSError + with pytest.raises(NePSError, match="pipeline space on disk does not match"): + neps.run( + evaluate_pipeline=eval_fn2, + pipeline_space=Space2(), + root_directory=str(root_dir), + evaluations_to_spend=2, + ) + + +def test_success_without_search_space_when_on_disk(tmp_path: Path): + """Test that not providing search space works when one exists on disk.""" + root_dir = tmp_path / "test_no_space" + + # Create initial state with TestSpace1 + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Continue WITHOUT providing pipeline_space - should load from disk + neps.run( + evaluate_pipeline=eval_fn1, + # pipeline_space not provided! + root_directory=str(root_dir), + evaluations_to_spend=3, # Total evaluations wanted + ) + + # Verify we have at least 2 evaluations (continuation worked) + df, _summary = neps.status(str(root_dir), print_summary=False) + assert len(df) >= 2, f"Should have at least 2 evaluations, got {len(df)}" + + +def test_error_when_no_space_provided_and_none_on_disk(tmp_path: Path): + """Test that not providing search space errors when none exists on disk.""" + root_dir = tmp_path / "test_no_space_error" + + # Try to run WITHOUT providing pipeline_space and with no existing run + with pytest.raises(ValueError, match="pipeline_space is required"): + neps.run( + evaluate_pipeline=eval_fn1, + # pipeline_space not provided and root_dir doesn't exist! + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + +def test_load_only_does_not_validate(tmp_path: Path, caplog): + """Test that load_only=True does not validate search space.""" + root_dir = tmp_path / "test_load_only" + + # Create initial state + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Load with load_only - should not error or warn about validation + with caplog.at_level(logging.WARNING): + state = NePSState.create_or_load( + path=root_dir, + load_only=True, + ) + loaded_space = state.lock_and_get_search_space() + + # Should not have validation errors/warnings since load_only=True + assert not any( + "pipeline space on disk" in record.message.lower() for record in caplog.records + ) + + # Should have loaded the original space + assert loaded_space is not None + + +def test_load_config_with_wrong_space_raises_error(tmp_path: Path): + """Test that load_config with wrong pipeline_space raises an error.""" + root_dir = tmp_path / "test_load_config_error" + + # Create run with TestSpace1 + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Find a config file + config_dir = root_dir / "configs" + configs = [ + d for d in config_dir.iterdir() if d.is_dir() and not d.name.startswith(".") + ] + assert len(configs) > 0, "Should have at least one config" + + config_path = configs[0] / "config.yaml" + + # Try to load with wrong pipeline_space - should raise error + with pytest.raises(NePSError, match="pipeline_space provided does not match"): + neps.load_config(config_path=config_path, pipeline_space=Space2()) + + +def test_load_config_without_space_auto_loads(tmp_path: Path): + """Test that load_config without pipeline_space auto-loads from disk.""" + root_dir = tmp_path / "test_load_config_auto" + + # Create run + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Find a config file + config_dir = root_dir / "configs" + configs = [ + d for d in config_dir.iterdir() if d.is_dir() and not d.name.startswith(".") + ] + assert len(configs) > 0, "Should have at least one config" + + config_path = configs[0] / "config.yaml" + + # Load config without providing space - should auto-load from disk + config = neps.load_config(config_path=config_path) + + assert "x" in config, "Should have x parameter" + + +def test_ddp_runtime_loads_search_space(tmp_path: Path): + """Test that DDP runtime path also loads search space correctly.""" + root_dir = tmp_path / "test_ddp" + + # Create initial state with search space + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Simulate DDP path - just load_only (DDP doesn't create state) + state = NePSState.create_or_load(path=root_dir, load_only=True) + loaded_space = state.lock_and_get_search_space() + + assert loaded_space is not None, "DDP should be able to load search space" + # NePS converts PipelineSpace to SearchSpace internally + assert isinstance(loaded_space, PipelineSpace | SearchSpace), "Should load correctly" + + +def test_status_without_space_works(tmp_path: Path): + """Test that status works without explicit pipeline_space.""" + root_dir = tmp_path / "test_status_auto" + + # Create a run + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Status without pipeline_space - should work + df, _summary = neps.status(str(root_dir), print_summary=False) + assert len(df) > 0, "Should have results" + + +def test_status_handles_missing_search_space_gracefully(tmp_path: Path): + """Test that status doesn't crash if search space can't be loaded.""" + root_dir = tmp_path / "test_status_missing" + + # Create a run + neps.run( + evaluate_pipeline=eval_fn1, + pipeline_space=Space1(), + root_directory=str(root_dir), + evaluations_to_spend=1, + ) + + # Delete the search space file + search_space_file = root_dir / "pipeline_space.pkl" + if search_space_file.exists(): + search_space_file.unlink() + + # Status with print_summary=False should work even without search space + df, _summary = neps.status(str(root_dir), print_summary=False) + assert len(df) > 0, "Should still get results"