From c34cfa91f1c06c46c86f48ee5f5670b43dd024b6 Mon Sep 17 00:00:00 2001 From: donglinjy Date: Wed, 26 Feb 2020 22:57:26 +0800 Subject: [PATCH 1/5] error in orion status --- .gitignore | 3 +++ src/orion/core/cli/status.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index db370f1aa..e50eff2c3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ *.pkl *.lock +# OS generated files +.DS_Store + # StarUML documentation *.mdj diff --git a/src/orion/core/cli/status.py b/src/orion/core/cli/status.py index 89aa6d82b..398f70fb5 100644 --- a/src/orion/core/cli/status.py +++ b/src/orion/core/cli/status.py @@ -104,13 +104,19 @@ def get_experiments(args): Commandline arguments. """ - projection = {'name': 1, 'version': 1} + projection = {'name': 1, 'version': 1, 'refers': 1} query = {'name': args['name']} if args.get('name') else {} experiments = get_storage().fetch_experiments(query, projection) + if args['name']: + root_experiments = experiments + else: + root_experiments = [exp for exp in experiments + if exp['refers'].get('root_id', exp['_id']) == exp['_id']] + return [experiment_builder.build_view(name=exp['name'], version=exp.get('version', 1)) - for exp in experiments] + for exp in root_experiments] def _has_named_children(exp): From ccf47fe5a951be6e9bbf9cf0ac475ee8822cafe7 Mon Sep 17 00:00:00 2001 From: donglinjy Date: Thu, 27 Feb 2020 14:19:32 +0800 Subject: [PATCH 2/5] add test case to cover invalid search algorithm in experiment --- tests/functional/commands/conftest.py | 19 ++++++++++ .../commands/invalidalgo_experiment.yaml | 38 +++++++++++++++++++ .../commands/test_status_command.py | 29 ++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 tests/functional/commands/invalidalgo_experiment.yaml diff --git a/tests/functional/commands/conftest.py b/tests/functional/commands/conftest.py index 49c53dab6..5c84a55ad 100644 --- a/tests/functional/commands/conftest.py +++ b/tests/functional/commands/conftest.py @@ -96,6 +96,19 @@ def exp_config(): return exp_config +@pytest.fixture() +def invalidalgo_exp_config(): + """Load an example database.""" + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'invalidalgo_experiment.yaml')) as f: + exp_config = list(yaml.safe_load_all(f)) + + for config in exp_config[0]: + backward.populate_space(config) + + return exp_config + + @pytest.fixture(scope='session') def database(): """Return Mongo database object to test with example entries.""" @@ -127,6 +140,12 @@ def db_instance(null_db_instances): return db +@pytest.fixture +def only_invalidalgo_experiments_db(clean_db, database, invalidalgo_exp_config): + """Clean the database and insert only experiments.""" + database.experiments.insert_many(invalidalgo_exp_config[0]) + + @pytest.fixture def only_experiments_db(clean_db, database, exp_config): """Clean the database and insert only experiments.""" diff --git a/tests/functional/commands/invalidalgo_experiment.yaml b/tests/functional/commands/invalidalgo_experiment.yaml new file mode 100644 index 000000000..833cc2770 --- /dev/null +++ b/tests/functional/commands/invalidalgo_experiment.yaml @@ -0,0 +1,38 @@ +# Example `orion` database, first are the "primary keys" (there's also an +# implicit _id key here in MongoDB, but we cannot base our search based on that) +--- + +# Example of entries in `experiments` collection +# configurations: some are inferred automatically, +# but most depend on user's input to `orion` +- name: supernaedo2-dendi + + metadata: + user: dendi + datetime: 2017-11-22T20:00:00 + orion_version: XYZ + user_script: functional/demo/black_box.py + user_args: ["--encoding_layer~choices(['rnn', 'lstm', 'gru'])", "--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])"] + VCS: + type: git + is_dirty: False + HEAD_sha: "test" + active_branch: null + diff_sha: "diff" + refers: + root_id: supernaedo2-dendi + parent_id: null + adapter: [] + pool_size: 2 + max_trials: 1000 + working_dir: + algorithms: + invalidalgo: # this must be logged as `Experiment` would log it, complete specification + seed: null + value: 5 + scoring: 0 + judgement: ~ + suspend: False + done: False + producer: + strategy: NoParallelStrategy \ No newline at end of file diff --git a/tests/functional/commands/test_status_command.py b/tests/functional/commands/test_status_command.py index 0fb460ad8..06b13df6e 100644 --- a/tests/functional/commands/test_status_command.py +++ b/tests/functional/commands/test_status_command.py @@ -93,6 +93,35 @@ def test_experiment_wout_success_wout_ac(clean_db, single_without_success, capsy assert captured == expected +def test_experiment_number_same_list_status(clean_db, only_invalidalgo_experiments_db, + single_without_success, capsys): + """Test status and list command output the consistent number of experiments""" + orion.core.cli.main(['status']) + + captured = capsys.readouterr().out + + expected = """\ +test_single_exp-v1 +================== +status quantity +----------- ---------- +broken 1 +interrupted 1 +new 1 +reserved 1 +suspended 1 + + +""" + assert captured == expected + + orion.core.cli.main(['list']) + + captured = capsys.readouterr().out + print(captured) + assert captured == " test_single_exp-v1\n" + + def test_experiment_w_trials_wout_ac(clean_db, single_with_trials, capsys): """Test status with only one experiment and all trials.""" orion.core.cli.main(['status']) From f2d2acda5c5aa5a7937beb1a08a31dac38911426 Mon Sep 17 00:00:00 2001 From: donglinjy Date: Thu, 27 Feb 2020 17:42:42 +0800 Subject: [PATCH 3/5] show message when no result with orion list --- src/orion/core/cli/list.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/orion/core/cli/list.py b/src/orion/core/cli/list.py index 448a1ae89..7684fe3b8 100644 --- a/src/orion/core/cli/list.py +++ b/src/orion/core/cli/list.py @@ -46,6 +46,10 @@ def main(args): root_experiments = [exp for exp in experiments if exp['refers'].get('root_id', exp['_id']) == exp['_id']] + if not root_experiments: + print("No experiment found") + return + for root_experiment in root_experiments: root = experiment_builder.build_view(name=root_experiment['name'], version=root_experiment.get('version')).node From e3f57ffc9a3e9479806efe3c8ae0c1762ca9d615 Mon Sep 17 00:00:00 2001 From: donglinjy Date: Thu, 27 Feb 2020 17:52:24 +0800 Subject: [PATCH 4/5] update test cases --- tests/functional/commands/test_list_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/commands/test_list_command.py b/tests/functional/commands/test_list_command.py index 5b9442410..ed4bff4cc 100644 --- a/tests/functional/commands/test_list_command.py +++ b/tests/functional/commands/test_list_command.py @@ -13,7 +13,7 @@ def test_no_exp(monkeypatch, clean_db, capsys): captured = capsys.readouterr().out - assert captured == "" + assert captured == "No experiment found\n" def test_single_exp(clean_db, one_experiment, capsys): @@ -84,7 +84,7 @@ def test_no_exp_name(clean_db, three_experiments, monkeypatch, capsys): captured = capsys.readouterr().out - assert captured == "" + assert captured == "No experiment found\n" def test_exp_name(clean_db, three_experiments, monkeypatch, capsys): From 007e7fc7f77c16f6b1fa1a91c32ae439ab881f65 Mon Sep 17 00:00:00 2001 From: donglinjy Date: Fri, 28 Feb 2020 13:39:46 +0800 Subject: [PATCH 5/5] remove confusing test cases --- src/orion/core/cli/status.py | 2 +- tests/functional/commands/conftest.py | 19 ---------- .../commands/invalidalgo_experiment.yaml | 38 ------------------- .../commands/test_status_command.py | 4 +- 4 files changed, 3 insertions(+), 60 deletions(-) delete mode 100644 tests/functional/commands/invalidalgo_experiment.yaml diff --git a/src/orion/core/cli/status.py b/src/orion/core/cli/status.py index 398f70fb5..ff8c6a04e 100644 --- a/src/orion/core/cli/status.py +++ b/src/orion/core/cli/status.py @@ -68,7 +68,7 @@ def main(args): raise RuntimeError("Cannot fetch specific version of experiments with --collapse " "or --expand-versions.") - print_evc(filter(lambda e: e.refers.get('parent_id') is None, experiments), **args) + print_evc(experiments, **args) # pylint: disable=unused-argument diff --git a/tests/functional/commands/conftest.py b/tests/functional/commands/conftest.py index 5c84a55ad..49c53dab6 100644 --- a/tests/functional/commands/conftest.py +++ b/tests/functional/commands/conftest.py @@ -96,19 +96,6 @@ def exp_config(): return exp_config -@pytest.fixture() -def invalidalgo_exp_config(): - """Load an example database.""" - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'invalidalgo_experiment.yaml')) as f: - exp_config = list(yaml.safe_load_all(f)) - - for config in exp_config[0]: - backward.populate_space(config) - - return exp_config - - @pytest.fixture(scope='session') def database(): """Return Mongo database object to test with example entries.""" @@ -140,12 +127,6 @@ def db_instance(null_db_instances): return db -@pytest.fixture -def only_invalidalgo_experiments_db(clean_db, database, invalidalgo_exp_config): - """Clean the database and insert only experiments.""" - database.experiments.insert_many(invalidalgo_exp_config[0]) - - @pytest.fixture def only_experiments_db(clean_db, database, exp_config): """Clean the database and insert only experiments.""" diff --git a/tests/functional/commands/invalidalgo_experiment.yaml b/tests/functional/commands/invalidalgo_experiment.yaml deleted file mode 100644 index 833cc2770..000000000 --- a/tests/functional/commands/invalidalgo_experiment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Example `orion` database, first are the "primary keys" (there's also an -# implicit _id key here in MongoDB, but we cannot base our search based on that) ---- - -# Example of entries in `experiments` collection -# configurations: some are inferred automatically, -# but most depend on user's input to `orion` -- name: supernaedo2-dendi - - metadata: - user: dendi - datetime: 2017-11-22T20:00:00 - orion_version: XYZ - user_script: functional/demo/black_box.py - user_args: ["--encoding_layer~choices(['rnn', 'lstm', 'gru'])", "--decoding_layer~choices(['rnn', 'lstm_with_attention', 'gru'])"] - VCS: - type: git - is_dirty: False - HEAD_sha: "test" - active_branch: null - diff_sha: "diff" - refers: - root_id: supernaedo2-dendi - parent_id: null - adapter: [] - pool_size: 2 - max_trials: 1000 - working_dir: - algorithms: - invalidalgo: # this must be logged as `Experiment` would log it, complete specification - seed: null - value: 5 - scoring: 0 - judgement: ~ - suspend: False - done: False - producer: - strategy: NoParallelStrategy \ No newline at end of file diff --git a/tests/functional/commands/test_status_command.py b/tests/functional/commands/test_status_command.py index 06b13df6e..f38ffb44a 100644 --- a/tests/functional/commands/test_status_command.py +++ b/tests/functional/commands/test_status_command.py @@ -93,7 +93,7 @@ def test_experiment_wout_success_wout_ac(clean_db, single_without_success, capsy assert captured == expected -def test_experiment_number_same_list_status(clean_db, only_invalidalgo_experiments_db, +def test_experiment_number_same_list_status(clean_db, single_without_success, capsys): """Test status and list command output the consistent number of experiments""" orion.core.cli.main(['status']) @@ -118,7 +118,7 @@ def test_experiment_number_same_list_status(clean_db, only_invalidalgo_experimen orion.core.cli.main(['list']) captured = capsys.readouterr().out - print(captured) + assert captured == " test_single_exp-v1\n"