From ec2d11f300c2b24c8202dcf29f41eea90db55439 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Thu, 28 Jul 2022 17:31:34 -0400 Subject: [PATCH 1/7] Bump parlai --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9c4e8d24f..8cbabaaf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ yappi>=1.2.5 torch>=1.5.0 tornado>=6.0.4 pandas>=1.0.3 -parlai>=0.1.20200716 +parlai>=1.6.0 psycopg2-binary>=2.8.5 pytest>=6.0.0 pyzmq>=19.0.1 From 9a5589a644333a139701c0a67fed2b3011b498ad Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Thu, 28 Jul 2022 17:38:10 -0400 Subject: [PATCH 2/7] soften pytest --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8cbabaaf2..4a1ff34d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ tornado>=6.0.4 pandas>=1.0.3 parlai>=1.6.0 psycopg2-binary>=2.8.5 -pytest>=6.0.0 +pytest>=5.0.0 pyzmq>=19.0.1 tqdm>=4.48.0 hydra-core>=1.0.0 From ae9176930bc0a2cd0b77575417834412bb4b07b3 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Thu, 28 Jul 2022 17:46:55 -0400 Subject: [PATCH 3/7] Test soul messing up testing --- light/world/souls/README.md | 2 +- light/world/souls/test_soul.py | 46 --------------------------- light/world/souls/tests/test_souls.py | 2 +- 3 files changed, 2 insertions(+), 48 deletions(-) delete mode 100644 light/world/souls/test_soul.py diff --git a/light/world/souls/README.md b/light/world/souls/README.md index a0e049e73..0ba99c6ed 100644 --- a/light/world/souls/README.md +++ b/light/world/souls/README.md @@ -23,4 +23,4 @@ Current flags: - [**`OnEventSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/on_event_soul.py): Extended `ModelSoul` class that is able to handle some basic scripted heuristic events as well as the more general `on_event` types that can be linked to a `GraphNode`. - [**`PlayerSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/player_soul.py): Simple `Soul` class that allows for a human player to take control of the `GraphAgent`. Requires use of a `PlayerProvider` containing the abstractions for sending observations and receiving actions. - [**`RepeatSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/repeat_soul.py): Bare-bones `Soul` class for use in demonstrations. Simply speaks back the observations it sees with a `SayEvent`. -- [**`TestSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/test_soul.py): Core soul for being able to run tests on the LIGHT `World` classes. Has a directly accessible `observations` field and a `do_act` function allowing scripted execution of actions. +- [**`TestSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/_test_soul.py): Core soul for being able to run tests on the LIGHT `World` classes. Has a directly accessible `observations` field and a `do_act` function allowing scripted execution of actions. diff --git a/light/world/souls/test_soul.py b/light/world/souls/test_soul.py deleted file mode 100644 index 45c26c2ff..000000000 --- a/light/world/souls/test_soul.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Facebook, Inc. and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from light.world.souls.soul import Soul -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from light.graph.elements.graph_nodes import GraphAgent - from light.graph.world.world import World - from light.graph.events.base import GraphEvent - - -class TestSoul(Soul): - """ - A Soul for use in testing - """ - - def __init__(self, target_node: "GraphAgent", world: "World"): - """ - TestSouls are created for test cases to take actions - and report observations - """ - super().__init__(target_node, world) - self.observations = [] - - def do_act(self, event): - """ - Handle executing an act on the given world - """ - event.execute(self.world) - - async def observe_event(self, event: "GraphEvent"): - """ - TestSouls do very little beyond saying what they observed, and smiling - for good measure. - """ - self.observations.append(event) - - def reap(self): - """ - TestSouls don't have any extra resources, and thus don't need to clean up. - """ - super().reap() diff --git a/light/world/souls/tests/test_souls.py b/light/world/souls/tests/test_souls.py index 2330c6c52..c3d1e174f 100644 --- a/light/world/souls/tests/test_souls.py +++ b/light/world/souls/tests/test_souls.py @@ -12,7 +12,7 @@ from light.graph.structured_graph import OOGraph from light.world.world import World from light.graph.events.graph_events import EmoteEvent, SayEvent -from light.world.souls.test_soul import TestSoul +from light.world.souls._test_soul import TestSoul from light.world.souls.repeat_soul import RepeatSoul From ecdce191ae1ae52a2544f3e83b5cb08acee7dedc Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Fri, 29 Jul 2022 11:30:51 -0400 Subject: [PATCH 4/7] Another mis-test --- deploy/web/server/tests/test_tornado_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/web/server/tests/test_tornado_server.py b/deploy/web/server/tests/test_tornado_server.py index 0f3c81d29..dbd301ce3 100644 --- a/deploy/web/server/tests/test_tornado_server.py +++ b/deploy/web/server/tests/test_tornado_server.py @@ -63,7 +63,7 @@ URL = f"http://localhost:{PORT}" -class TestFlags: +class MockFlags: def __init__(self, hostname, port): self.hostname = hostname self.port = port @@ -79,7 +79,7 @@ def setUp(self): # Need to fix this somehow... self.db_path = os.path.join(self.data_dir, "test_server.db") self.db = LIGHTDatabase(self.db_path) - self.FLAGS = TestFlags("localhost", PORT) + self.FLAGS = MockFlags("localhost", PORT) self.client = httpclient.AsyncHTTPClient() super().setUp() From 012c2aed34d4449ae738bc67db1000c9796db0b6 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Fri, 29 Jul 2022 12:01:29 -0400 Subject: [PATCH 5/7] Accidentally dropped file --- light/world/souls/_test_soul.py | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 light/world/souls/_test_soul.py diff --git a/light/world/souls/_test_soul.py b/light/world/souls/_test_soul.py new file mode 100644 index 000000000..45c26c2ff --- /dev/null +++ b/light/world/souls/_test_soul.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +# Copyright (c) Facebook, Inc. and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from light.world.souls.soul import Soul +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from light.graph.elements.graph_nodes import GraphAgent + from light.graph.world.world import World + from light.graph.events.base import GraphEvent + + +class TestSoul(Soul): + """ + A Soul for use in testing + """ + + def __init__(self, target_node: "GraphAgent", world: "World"): + """ + TestSouls are created for test cases to take actions + and report observations + """ + super().__init__(target_node, world) + self.observations = [] + + def do_act(self, event): + """ + Handle executing an act on the given world + """ + event.execute(self.world) + + async def observe_event(self, event: "GraphEvent"): + """ + TestSouls do very little beyond saying what they observed, and smiling + for good measure. + """ + self.observations.append(event) + + def reap(self): + """ + TestSouls don't have any extra resources, and thus don't need to clean up. + """ + super().reap() From c8b05fa70ff16dfada87e2a55e6ba436b1390179 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Fri, 29 Jul 2022 12:09:33 -0400 Subject: [PATCH 6/7] Stop confusing the tests --- light/world/souls/README.md | 2 +- .../souls/{_test_soul.py => mock_soul.py} | 8 ++--- light/world/souls/tests/test_souls.py | 30 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) rename light/world/souls/{_test_soul.py => mock_soul.py} (83%) diff --git a/light/world/souls/README.md b/light/world/souls/README.md index 0ba99c6ed..2519aa763 100644 --- a/light/world/souls/README.md +++ b/light/world/souls/README.md @@ -23,4 +23,4 @@ Current flags: - [**`OnEventSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/on_event_soul.py): Extended `ModelSoul` class that is able to handle some basic scripted heuristic events as well as the more general `on_event` types that can be linked to a `GraphNode`. - [**`PlayerSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/player_soul.py): Simple `Soul` class that allows for a human player to take control of the `GraphAgent`. Requires use of a `PlayerProvider` containing the abstractions for sending observations and receiving actions. - [**`RepeatSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/repeat_soul.py): Bare-bones `Soul` class for use in demonstrations. Simply speaks back the observations it sees with a `SayEvent`. -- [**`TestSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/_test_soul.py): Core soul for being able to run tests on the LIGHT `World` classes. Has a directly accessible `observations` field and a `do_act` function allowing scripted execution of actions. +- [**`MockSoul`**](https://github.com/facebookresearch/LIGHT/tree/main/light/world/souls/mock_soul.py): Core soul for being able to run tests on the LIGHT `World` classes. Has a directly accessible `observations` field and a `do_act` function allowing scripted execution of actions. diff --git a/light/world/souls/_test_soul.py b/light/world/souls/mock_soul.py similarity index 83% rename from light/world/souls/_test_soul.py rename to light/world/souls/mock_soul.py index 45c26c2ff..dfa1579f2 100644 --- a/light/world/souls/_test_soul.py +++ b/light/world/souls/mock_soul.py @@ -13,14 +13,14 @@ from light.graph.events.base import GraphEvent -class TestSoul(Soul): +class MockSoul(Soul): """ A Soul for use in testing """ def __init__(self, target_node: "GraphAgent", world: "World"): """ - TestSouls are created for test cases to take actions + MockSouls are created for test cases to take actions and report observations """ super().__init__(target_node, world) @@ -34,13 +34,13 @@ def do_act(self, event): async def observe_event(self, event: "GraphEvent"): """ - TestSouls do very little beyond saying what they observed, and smiling + MockSouls do very little beyond saying what they observed, and smiling for good measure. """ self.observations.append(event) def reap(self): """ - TestSouls don't have any extra resources, and thus don't need to clean up. + MockSouls don't have any extra resources, and thus don't need to clean up. """ super().reap() diff --git a/light/world/souls/tests/test_souls.py b/light/world/souls/tests/test_souls.py index c3d1e174f..003cc98e2 100644 --- a/light/world/souls/tests/test_souls.py +++ b/light/world/souls/tests/test_souls.py @@ -12,7 +12,7 @@ from light.graph.structured_graph import OOGraph from light.world.world import World from light.graph.events.graph_events import EmoteEvent, SayEvent -from light.world.souls._test_soul import TestSoul +from light.world.souls.mock_soul import MockSoul from light.world.souls.repeat_soul import RepeatSoul @@ -26,7 +26,7 @@ def wrapper(*args, **kwargs): return wrapper -class TestSouls(unittest.TestCase): +class MockSouls(unittest.TestCase): """Unit tests for simple souls""" def test_init_soul(self): @@ -37,16 +37,16 @@ def test_init_soul(self): agent_node.force_move_to(room_node) test_world = World({}, None, True) test_world.oo_graph = test_graph - test_soul = TestSoul(agent_node, test_world) - self.assertEqual(agent_node, test_soul.target_node) - self.assertEqual(test_world, test_soul.world) + mock_soul = MockSoul(agent_node, test_world) + self.assertEqual(agent_node, mock_soul.target_node) + self.assertEqual(test_world, mock_soul.world) test_event = EmoteEvent.construct_from_args( agent_node, targets=[], text="smile" ) - test_soul.do_act(test_event) + mock_soul.do_act(test_event) - test_soul.reap() + mock_soul.reap() @async_test async def test_message_sending(self): @@ -65,7 +65,7 @@ async def test_message_sending(self): test_world.oo_graph = test_graph purgatory = test_world.purgatory - purgatory.register_filler_soul_provider("test", TestSoul, lambda: []) + purgatory.register_filler_soul_provider("test", MockSoul, lambda: []) purgatory.register_filler_soul_provider("repeat", RepeatSoul, lambda: []) purgatory.fill_soul(test_node, "test") @@ -74,23 +74,23 @@ async def test_message_sending(self): purgatory.fill_soul(repeat_node, "repeat") self.assertIn(repeat_node.node_id, purgatory.node_id_to_soul) - test_soul: "TestSoul" = purgatory.node_id_to_soul[test_node.node_id] + mock_soul: "MockSoul" = purgatory.node_id_to_soul[test_node.node_id] repeat_soul = purgatory.node_id_to_soul[repeat_node.node_id] # Make the test soul act, and observe what follows test_event = EmoteEvent.construct_from_args(test_node, targets=[], text="smile") - test_soul.do_act(test_event) + mock_soul.do_act(test_event) - observations = test_soul.observations + observations = mock_soul.observations start_time = time.time() OBSERVATION_WAIT_TIMEOUT = 3.3 while len(observations) < 3: self.assertTrue( time.time() - start_time < OBSERVATION_WAIT_TIMEOUT, f"Exceeded expected duration {OBSERVATION_WAIT_TIMEOUT} waiting " - f"for parrot events, found {test_soul.observations}", + f"for parrot events, found {mock_soul.observations}", ) - observations = test_soul.observations + observations = mock_soul.observations await asyncio.sleep(0.01) # Observations should be the self smile event, then the repeat agent's say and smile @@ -114,11 +114,11 @@ async def test_message_sending(self): self.assertEqual(other_emote_observe.actor, repeat_node) self.assertEqual(other_emote_observe.text_content, "smile") - observations = test_soul.observations + observations = mock_soul.observations # Extra observation may have slipped in? self.assertEqual(len(observations), 3, "Unexpected amount of observations") self.assertEqual( - len(test_soul._observe_futures), 0, "All obs threads should have deleted" + len(mock_soul._observe_futures), 0, "All obs threads should have deleted" ) From 1a6afd9336069ca4ac737ac1863db1916da52f79 Mon Sep 17 00:00:00 2001 From: Jack Urbanek Date: Fri, 29 Jul 2022 14:28:14 -0400 Subject: [PATCH 7/7] another one snuck by --- .../collect_narrations/{test_get.py => try_get.py} | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename crowdsourcing/custom_world_interactions/collect_narrations/{test_get.py => try_get.py} (85%) diff --git a/crowdsourcing/custom_world_interactions/collect_narrations/test_get.py b/crowdsourcing/custom_world_interactions/collect_narrations/try_get.py similarity index 85% rename from crowdsourcing/custom_world_interactions/collect_narrations/test_get.py rename to crowdsourcing/custom_world_interactions/collect_narrations/try_get.py index debf78dd8..e4a9b2619 100644 --- a/crowdsourcing/custom_world_interactions/collect_narrations/test_get.py +++ b/crowdsourcing/custom_world_interactions/collect_narrations/try_get.py @@ -15,9 +15,11 @@ DO_REVIEW = True -units = mephisto_data_browser.get_units_for_task_name("objects-interaction-task-pilot-5") +units = mephisto_data_browser.get_units_for_task_name( + "objects-interaction-task-pilot-5" +) print(f"prev len: {len(units)}") print(Counter([u.get_status() for u in units])) units = [u for u in units if u.get_status() == "completed"] -print(f"len: {len(units)}") \ No newline at end of file +print(f"len: {len(units)}")