Agnext websurfer (#205)

* Initial work on multimodal websurfer

* A little more progress.

* Getting function calling to work.

* Some basic progress with navigation.

* Added ability to print multimodal messages to console.

* Fixed hatch error

* Nicely print multimodal messages to console.

* Got OCR working.

* Fixed the click action.

* Solved some hatch errors.

* Fixed some formatting errors.

* Fixed more type errors.

* Yet more fixes to types.

* Fixed many type errors.

* Fixed all type errors. Some needed to be ignored. See todos.

* Fixed all? hatch errors?

* Fixed multiline aria-names in prompts.

Author: afourney

python/teams/team-one/examples/example_websurfer.py

@@ -0,0 +1,50 @@
+import asyncio
+import logging
+
+from agnext.application import SingleThreadedAgentRuntime
+from agnext.application.logging import EVENT_LOGGER_NAME
+from team_one.agents.multimodal_web_surfer import MultimodalWebSurfer
+from team_one.agents.orchestrator import RoundRobinOrchestrator
+from team_one.agents.user_proxy import UserProxy
+from team_one.messages import RequestReplyMessage
+from team_one.utils import LogHandler, create_completion_client_from_env
+
+# NOTE: Don't forget to 'playwright install --with-deps chromium'
+
+
+async def main() -> None:
+    # Create the runtime.
+    runtime = SingleThreadedAgentRuntime()
+
+    # Create an appropriate client
+    client = create_completion_client_from_env()
+
+    # Register agents.
+    web_surfer = runtime.register_and_get_proxy(
+        "WebSurfer",
+        lambda: MultimodalWebSurfer(),
+    )
+
+    user_proxy = runtime.register_and_get_proxy(
+        "UserProxy",
+        lambda: UserProxy(),
+    )
+
+    runtime.register("orchestrator", lambda: RoundRobinOrchestrator([web_surfer, user_proxy]))
+
+    run_context = runtime.start()
+
+    actual_surfer = runtime._get_agent(web_surfer.id)  # type: ignore
+    assert isinstance(actual_surfer, MultimodalWebSurfer)
+    await actual_surfer.init(model_client=client, browser_channel="chromium")
+
+    await runtime.send_message(RequestReplyMessage(), user_proxy.id)
+    await run_context.stop_when_idle()
+
+
+if __name__ == "__main__":
+    logger = logging.getLogger(EVENT_LOGGER_NAME)
+    logger.setLevel(logging.INFO)
+    log_handler = LogHandler()
+    logger.handlers = [log_handler]
+    asyncio.run(main())

python/teams/team-one/pyproject.toml

@@ -17,7 +17,8 @@ classifiers = [
 ]
 dependencies = [
     "agnext@{root:parent:parent:uri}",
-    "aiofiles"
+    "aiofiles",
+    "playwright"
 ]
 
 [tool.hatch.envs.default]
@@ -53,7 +54,7 @@ allow-direct-references = true
 [tool.ruff]
 line-length = 120
 fix = true
-exclude = ["build", "dist"]
+exclude = ["build", "dist", "page_script.js"]
 target-version = "py310"
 include = ["src/**", "examples/*.py"]
 

python/teams/team-one/src/team_one/agents/multimodal_web_surfer/__init__.py

@@ -0,0 +1,3 @@
+from .multimodal_web_surfer import MultimodalWebSurfer
+
+__all__ = ("MultimodalWebSurfer",)