From 45001ace35b4419c34f72bf12380281e0d79ba94 Mon Sep 17 00:00:00 2001 From: Xavier Bouthillier Date: Thu, 29 Aug 2019 15:53:07 -0400 Subject: [PATCH] Avoid branching in `info` Why: When something is different in the environment, the experiment may branch since we are not using a `View` in `info`. We should rather use a view but for now they don't have access to instantiated space and algorithms which are required by `info`. How: Use a view and reconfigure it to instantiate the space and algo. This is not done by default in `View` because they crash if the users scripts in the experiment are not present on the current FS. At least now this only happens in `info`, which was already the case anyway since we were not using a `View`. --- src/orion/core/cli/info.py | 22 +++++++++++++-- src/orion/storage/base.py | 4 ++- tests/functional/commands/conftest.py | 12 ++++++++ .../functional/commands/test_info_command.py | 22 +++++++++++++-- tests/unittests/core/cli/test_info.py | 28 +++++++++++++++++-- 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/orion/core/cli/info.py b/src/orion/core/cli/info.py index 07297a060..299d3343c 100755 --- a/src/orion/core/cli/info.py +++ b/src/orion/core/cli/info.py @@ -10,6 +10,7 @@ """ import logging +import sys from orion.core.cli.base import get_basic_args_group from orion.core.io.evc_builder import EVCBuilder @@ -27,9 +28,23 @@ def add_subparser(parser): return info_parser +# pylint: disable=protected-access +def hack_until_config_is_refactored(experiment): + """Build the space and the algorithm""" + experiment._experiment._instantiate_config(experiment.configuration) + experiment._experiment._init_done = True + + def main(args): """Fetch config and info experiments""" - experiment = EVCBuilder().build_from(args) + try: + experiment = EVCBuilder().build_view_from(args) + except ValueError: + print('Experiment {} not found in db.'.format(args.get('name', None))) + sys.exit(1) + + hack_until_config_is_refactored(experiment) + print(format_info(experiment)) @@ -326,6 +341,8 @@ def format_space(experiment): user: {experiment.metadata[user]} datetime: {experiment.metadata[datetime]} orion version: {experiment.metadata[orion_version]} +VCS: +{vcs} """ @@ -333,7 +350,8 @@ def format_metadata(experiment): """Render a string for metadata section""" metadata_string = METADATA_TEMPLATE.format( title=format_title("Meta-data"), - experiment=experiment) + experiment=experiment, + vcs=format_dict(experiment.metadata.get('VCS', {}), depth=1, width=2)) return metadata_string diff --git a/src/orion/storage/base.py b/src/orion/storage/base.py index bd8bf4a8d..df7c060d6 100644 --- a/src/orion/storage/base.py +++ b/src/orion/storage/base.py @@ -247,13 +247,15 @@ class ReadOnlyStorageProtocol(object): __slots__ = ('_storage', ) valid_attributes = { + 'get_trial', 'fetch_trials', 'fetch_experiments', 'count_broken_trials', 'count_completed_trials', 'fetch_noncompleted_trials', 'fetch_pending_trials', - 'fetch_lost_trials' + 'fetch_lost_trials', + 'fetch_trial_by_status' } def __init__(self, protocol): diff --git a/tests/functional/commands/conftest.py b/tests/functional/commands/conftest.py index d15f778d6..e260dcf71 100644 --- a/tests/functional/commands/conftest.py +++ b/tests/functional/commands/conftest.py @@ -155,6 +155,18 @@ def one_experiment(monkeypatch, db_instance): return get_storage().fetch_experiments({'name': name})[0] +@pytest.fixture +def one_experiment_changed_vcs(one_experiment): + """Create an experiment without trials.""" + experiment = ExperimentBuilder().build_from({'name': one_experiment['name']}) + + experiment.metadata['VCS'] = { + 'type': 'git', 'is_dirty': False, 'HEAD_sha': 'new', 'active_branch': 'master', + 'diff_sha': None} + + get_storage().update_experiment(experiment, metadata=experiment.metadata) + + @pytest.fixture def one_experiment_no_version(monkeypatch, one_experiment): """Create an experiment without trials.""" diff --git a/tests/functional/commands/test_info_command.py b/tests/functional/commands/test_info_command.py index c6bf712c3..4cf30a4bb 100644 --- a/tests/functional/commands/test_info_command.py +++ b/tests/functional/commands/test_info_command.py @@ -1,16 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """Perform a functional test of the info command.""" +import pytest + import orion.core.cli +import orion.core.io.resolve_config def test_info_no_hit(clean_db, one_experiment, capsys): """Test info if no experiment with given name.""" - orion.core.cli.main(['info', '--name', 'i do not exist']) + with pytest.raises(SystemExit) as exc: + orion.core.cli.main(['info', '--name', 'i do not exist']) + + assert str(exc.value) == '1' captured = capsys.readouterr().out - assert captured == 'Error: No commandline configuration found for new experiment.\n' + assert captured == 'Experiment i do not exist not found in db.\n' def test_info_hit(clean_db, one_experiment, capsys): @@ -29,3 +35,15 @@ def test_info_broken(clean_db, broken_refers, capsys): captured = capsys.readouterr().out assert '--x~uniform(0,1)' in captured + + +def test_info_no_branching(clean_db, one_experiment_changed_vcs, capsys): + """Test info if config file is different + + Version should not increase! + """ + orion.core.cli.main(['info', '--name', 'test_single_exp']) + + captured = capsys.readouterr().out + + assert '\nversion: 1\n' in captured diff --git a/tests/unittests/core/cli/test_info.py b/tests/unittests/core/cli/test_info.py index a4c70c9ec..92a80c8d0 100755 --- a/tests/unittests/core/cli/test_info.py +++ b/tests/unittests/core/cli/test_info.py @@ -440,13 +440,25 @@ def test_format_metadata(): experiment.metadata = dict( user='user', datetime='now', - orion_version='1.0.1') + orion_version='1.0.1', + VCS=dict( + HEAD_sha='sha', + active_branch='branch', + diff_sha='smt', + is_dirty=True, + type='git')) assert format_metadata(experiment) == """\ Meta-data ========= user: user datetime: now orion version: 1.0.1 +VCS: + HEAD_sha: sha + active_branch: branch + diff_sha: smt + is_dirty: True + type: git """ @@ -569,7 +581,13 @@ def test_format_info(algorithm_dict, dummy_trial): experiment.metadata.update(dict( user='user', datetime='now', - orion_version='1.0.1')) + orion_version='1.0.1', + VCS=dict( + HEAD_sha='sha', + active_branch='branch', + diff_sha='smt', + is_dirty=True, + type='git'))) ROOT_NAME = 'root-name' PARENT_NAME = 'parent-name' @@ -639,6 +657,12 @@ def test_format_info(algorithm_dict, dummy_trial): user: user datetime: now orion version: 1.0.1 +VCS: + HEAD_sha: sha + active_branch: branch + diff_sha: smt + is_dirty: True + type: git Parent experiment