Skip to content

Commit

Permalink
Merge pull request #630 from bouthilx/feature/filter_duplicate_in_evc
Browse files Browse the repository at this point in the history 
Feature/filter duplicate in evc
  • Loading branch information
@bouthilx
bouthilx authored Jul 29, 2021
2 parents 4d89c20 + cd5f357 commit 0921b7f
Show file tree
Hide file tree
Showing 7 changed files with 705 additions and 57 deletions.
4 changes: 4 additions & 0 deletions src/orion/core/evc/adapters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import copy
from abc import ABCMeta, abstractmethod

from orion.algo.space import Dimension
from orion.core.io.space_builder import DimensionBuilder
from orion.core.utils import Factory
from orion.core.worker.trial import Trial
Expand Down Expand Up @@ -278,6 +279,9 @@ def forward(self, trials):
:meth:`orion.core.evc.adapters.BaseAdapter.forward`
"""
if self.param.value is Dimension.NO_DEFAULT_VALUE:
return []

adapted_trials = []

for trial in trials:
Expand Down
47 changes: 45 additions & 2 deletions src/orion/core/evc/experiment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
analyzing an EVC tree.
"""
import functools
import logging

from orion.core.evc.tree import TreeNode
Expand Down Expand Up @@ -192,22 +193,41 @@ def retrieve_trials(node, parent_or_children):
children_trials.set_parent(parent_trials)

adapt_trials(children_trials)

return sum([node.item["trials"] for node in children_trials.root], [])


def _adapt_parent_trials(node, parent_trials_node):
def _adapt_parent_trials(node, parent_trials_node, ids):
"""Adapt trials from the parent recursively
.. note::
To call with node.map(fct, node.parent) to connect with parents
"""
# Ids from children are passed to prioritized them if they are also present in parent nodes.
node_ids = (
set(
trial.compute_trial_hash(trial, ignore_lie=True, ignore_experiment=True)
for trial in node.item["trials"]
)
| ids
)
if parent_trials_node is not None:
adapter = node.item["experiment"].refers["adapter"]
for parent in parent_trials_node.root:
parent.item["trials"] = adapter.forward(parent.item["trials"])

# if trial is in current exp, filter out
parent.item["trials"] = [
trial
for trial in parent.item["trials"]
if trial.compute_trial_hash(
trial, ignore_lie=True, ignore_experiment=True
)
not in node_ids
]

return node.item, parent_trials_node


Expand All @@ -219,15 +239,38 @@ def _adapt_children_trials(node, children_trials_nodes):
To call with node.map(fct, node.children) to connect with children
"""
ids = set(
trial.compute_trial_hash(trial, ignore_lie=True, ignore_experiment=True)
for trial in node.item["trials"]
)

for child in children_trials_nodes:
adapter = child.item["experiment"].refers["adapter"]
for subchild in child: # Includes child itself
subchild.item["trials"] = adapter.backward(subchild.item["trials"])

# if trial is in current node, filter out
subchild.item["trials"] = [
trial
for trial in subchild.item["trials"]
if trial.compute_trial_hash(
trial, ignore_lie=True, ignore_experiment=True
)
not in ids
]

return node.item, children_trials_nodes


def adapt_trials(trials_tree):
"""Adapt trials recursively so that they are all compatible with current experiment."""
trials_tree.map(_adapt_parent_trials, trials_tree.parent)
trials_tree.map(_adapt_children_trials, trials_tree.children)
ids = set()
for child in trials_tree.children:
for trial in child.item["trials"]:
ids.add(
trial.compute_trial_hash(trial, ignore_lie=True, ignore_experiment=True)
)
trials_tree.map(
functools.partial(_adapt_parent_trials, ids=ids), trials_tree.parent
)
8 changes: 4 additions & 4 deletions tests/functional/commands/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ def family_with_trials(two_experiments):
x["value"] = x_value
y["value"] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x["value"] = x_value
x["value"] = x_value + 0.5 # To avoid duplicates
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write("trials", trial.to_dict())
Expand Down Expand Up @@ -262,7 +262,7 @@ def three_family_with_trials(three_experiments_family, family_with_trials):

x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
x["value"] = x_value + 0.75 # To avoid duplicates
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, z], status=status)
x_value += 1
Expand Down Expand Up @@ -305,7 +305,7 @@ def three_family_branch_with_trials(

x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
x["value"] = x_value + 0.25 # To avoid duplicates
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, y, z], status=status)
Expand Down Expand Up @@ -424,7 +424,7 @@ def three_experiments_same_name_with_trials(two_experiments_same_name, storage):
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
x["value"] = x_value + 0.1 # To avoid duplicates
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x], status=status)
Expand Down
84 changes: 42 additions & 42 deletions tests/functional/commands/test_status_command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,12 +501,12 @@ def test_two_related_w_a_wout_c(family_with_trials, capsys):
========================
id status
-------------------------------- -----------
890b4f07685ed020f5d9e28cac9316e1 broken
ff81ff46da5ffe6bd623fb38a06df993 completed
78a3e60699eee1d0b9bc51a049168fce interrupted
13cd454155748351790525e3079fb620 new
d1b7ecbd3621de9195a42c76defb6603 reserved
33d6208ef03cb236a8f3b567665c357d suspended
2b08bdbad673e60fef739b7f162d4120 broken
af45e26f349f9b186e5c05a91d854fb5 completed
b334ea9e2c86873ddb18b206cf72cc27 interrupted
16b024079173ca3903eb956c478afa3d new
17ce012a15a7398d3e7703d0c13e21c2 reserved
d4721fe7f50df1fe3ba60424df6dec67 suspended
"""
Expand Down Expand Up @@ -537,12 +537,12 @@ def test_three_unrelated_w_a_wout_c(three_experiments_with_trials, capsys):
========================
id status
-------------------------------- -----------
890b4f07685ed020f5d9e28cac9316e1 broken
ff81ff46da5ffe6bd623fb38a06df993 completed
78a3e60699eee1d0b9bc51a049168fce interrupted
13cd454155748351790525e3079fb620 new
d1b7ecbd3621de9195a42c76defb6603 reserved
33d6208ef03cb236a8f3b567665c357d suspended
2b08bdbad673e60fef739b7f162d4120 broken
af45e26f349f9b186e5c05a91d854fb5 completed
b334ea9e2c86873ddb18b206cf72cc27 interrupted
16b024079173ca3903eb956c478afa3d new
17ce012a15a7398d3e7703d0c13e21c2 reserved
d4721fe7f50df1fe3ba60424df6dec67 suspended
test_single_exp-v1
Expand Down Expand Up @@ -585,24 +585,24 @@ def test_three_related_w_a_wout_c(three_family_with_trials, capsys):
========================
id status
-------------------------------- -----------
890b4f07685ed020f5d9e28cac9316e1 broken
ff81ff46da5ffe6bd623fb38a06df993 completed
78a3e60699eee1d0b9bc51a049168fce interrupted
13cd454155748351790525e3079fb620 new
d1b7ecbd3621de9195a42c76defb6603 reserved
33d6208ef03cb236a8f3b567665c357d suspended
2b08bdbad673e60fef739b7f162d4120 broken
af45e26f349f9b186e5c05a91d854fb5 completed
b334ea9e2c86873ddb18b206cf72cc27 interrupted
16b024079173ca3903eb956c478afa3d new
17ce012a15a7398d3e7703d0c13e21c2 reserved
d4721fe7f50df1fe3ba60424df6dec67 suspended
test_double_exp_child2-v1
=========================
id status
-------------------------------- -----------
1c238040d6b6d8423d99a08551fe0998 broken
2c13424a9212ab92ea592bdaeb1c13e9 completed
a2680fbda1faa9dfb94946cf25536f44 interrupted
abbda454d0577ded5b8e784a9d6d5abb new
df58aa8fd875f129f7faa84eb15ca453 reserved
71657e86bad0f2e8b06098a64cb883b6 suspended
f736224f9687f86c493a004696abd95b broken
2def838a2eb199820f283e1948e7c37a completed
75752e1ba3c9007e42616249087a7fef interrupted
5f5e1c8d886ef0b0c0666d6db7bf1723 new
2623a01bd2483a5e18fac9bc3dfbdee2 reserved
2eecad70c53bb52c99efad36f2d9502f suspended
"""
Expand Down Expand Up @@ -633,24 +633,24 @@ def test_three_related_branch_w_a_wout_c(three_family_branch_with_trials, capsys
========================
id status
-------------------------------- -----------
890b4f07685ed020f5d9e28cac9316e1 broken
ff81ff46da5ffe6bd623fb38a06df993 completed
78a3e60699eee1d0b9bc51a049168fce interrupted
13cd454155748351790525e3079fb620 new
d1b7ecbd3621de9195a42c76defb6603 reserved
33d6208ef03cb236a8f3b567665c357d suspended
2b08bdbad673e60fef739b7f162d4120 broken
af45e26f349f9b186e5c05a91d854fb5 completed
b334ea9e2c86873ddb18b206cf72cc27 interrupted
16b024079173ca3903eb956c478afa3d new
17ce012a15a7398d3e7703d0c13e21c2 reserved
d4721fe7f50df1fe3ba60424df6dec67 suspended
test_double_exp_grand_child-v1
==============================
id status
-------------------------------- -----------
e374d8f802aed52c07763545f46228a7 broken
f9ee14ff9ef0b95ed7a24860731c85a9 completed
3f7dff101490727d5fa0efeb36ca6366 interrupted
40838d46dbf7778a3cb51b7a09118391 new
b7860a18b2700cce4e8009cde543975c reserved
cd406126bc350ad82ac77c75174cc8a2 suspended
e1c929d9c4d48eca4dcd463690e4096d broken
e8eec526a7f7fdea5e4a30d969ec69ae completed
e88f1d26158efb393ae1278c6ef115fe interrupted
4510dc7d16a692c7415dd2898faced9f new
523881db96da0de9dd972ef8f3545f81 reserved
f967423d15c50ddf88c242f511997ff7 suspended
"""
Expand Down Expand Up @@ -853,7 +853,7 @@ def test_two_related_w_ac(family_with_trials, capsys):
cbb766d729294f77f0ca86ff2bf72707 completed
ca6576848f17201852225d816fb71fcc interrupted
28097ba31dbdffc0aa265c6bc5c98b0f new
4c409da13bdc93c54f6997797c296356 new
cd16dd40955335aae3bd40371e636b71 new
adbe6c400cd1e667696e28fbecd000a0 reserved
5679af6c6bb54aa8042043008ab2bc1f suspended
Expand All @@ -878,7 +878,7 @@ def test_three_unrelated_w_ac(three_experiments_with_trials, capsys):
cbb766d729294f77f0ca86ff2bf72707 completed
ca6576848f17201852225d816fb71fcc interrupted
28097ba31dbdffc0aa265c6bc5c98b0f new
4c409da13bdc93c54f6997797c296356 new
cd16dd40955335aae3bd40371e636b71 new
adbe6c400cd1e667696e28fbecd000a0 reserved
5679af6c6bb54aa8042043008ab2bc1f suspended
Expand Down Expand Up @@ -915,8 +915,8 @@ def test_three_related_w_ac(three_family_with_trials, capsys):
cbb766d729294f77f0ca86ff2bf72707 completed
ca6576848f17201852225d816fb71fcc interrupted
28097ba31dbdffc0aa265c6bc5c98b0f new
4c409da13bdc93c54f6997797c296356 new
b97518f91e006cd4a2805657c596b11c new
cd16dd40955335aae3bd40371e636b71 new
8d5652cba225224d6702107e97a53cd9 new
adbe6c400cd1e667696e28fbecd000a0 reserved
5679af6c6bb54aa8042043008ab2bc1f suspended
Expand All @@ -941,8 +941,8 @@ def test_three_related_branch_w_ac(three_family_branch_with_trials, capsys):
cbb766d729294f77f0ca86ff2bf72707 completed
ca6576848f17201852225d816fb71fcc interrupted
28097ba31dbdffc0aa265c6bc5c98b0f new
4c409da13bdc93c54f6997797c296356 new
5183ee9c28601cc78c0a148a386df9f9 new
cd16dd40955335aae3bd40371e636b71 new
17fab2503ac14ae55e207c7cca1b8f1f new
adbe6c400cd1e667696e28fbecd000a0 reserved
5679af6c6bb54aa8042043008ab2bc1f suspended
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/demo/test_demo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,7 @@ def test_workon():
assert len(params) == 1
px = params["/x"]
assert isinstance(px, float)
assert (px - 34.56789) < 5
assert (px - 34.56789) < 20


def test_stress_unique_folder_creation(storage, monkeypatch, tmpdir, capfd):
Expand Down
18 changes: 10 additions & 8 deletions tests/functional/serving/test_trials_resource.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def add_experiment(**kwargs):
)


def add_trial(experiment: int, status: str = None, **kwargs):
def add_trial(experiment: int, status: str = None, value=10, **kwargs):
"""
Add trials to the dummy orion instance
Expand All @@ -80,6 +80,7 @@ def add_trial(experiment: int, status: str = None, **kwargs):
kwargs["status"] = status

base_trial.update(copy.deepcopy(kwargs))
base_trial["params"][0]["value"] = value
get_storage().register_trial(Trial(**base_trial))


Expand Down Expand Up @@ -181,9 +182,10 @@ def test_trials_for_all_versions(self, client):
add_experiment(name="a", version=2, _id=2)
add_experiment(name="a", version=3, _id=3)

add_trial(experiment=1, id_override="00")
add_trial(experiment=2, id_override="01")
add_trial(experiment=3, id_override="02")
# Specify values to avoid duplicates
add_trial(experiment=1, id_override="00", value=1)
add_trial(experiment=2, id_override="01", value=2)
add_trial(experiment=3, id_override="02", value=3)

# Happy case default
response = client.simulate_get("/trials/a?ancestors=true")
Expand Down Expand Up @@ -256,10 +258,10 @@ def test_trials_by_from_specific_version_by_status_with_ancestors(self, client):
add_experiment(name="a", version=2, _id=3)
add_experiment(name="a", version=3, _id=4)

add_trial(experiment=1, id_override="00", status="completed")
add_trial(experiment=3, id_override="01", status="broken")
add_trial(experiment=3, id_override="02", status="completed")
add_trial(experiment=2, id_override="03", status="completed")
add_trial(experiment=1, id_override="00", value=1, status="completed")
add_trial(experiment=3, id_override="01", value=2, status="broken")
add_trial(experiment=3, id_override="02", value=3, status="completed")
add_trial(experiment=2, id_override="03", value=4, status="completed")

response = client.simulate_get(
"/trials/a?ancestors=true&version=2&status=completed"
Expand Down
Loading

0 comments on commit 0921b7f

Please sign in to comment.