Skip to content

Commit b0ab6c1

Browse files
committed
Add headless browser to the WebSurferAgent, closes #1481
1 parent 26daa18 commit b0ab6c1

File tree

11 files changed

+648
-130
lines changed

11 files changed

+648
-130
lines changed

.github/workflows/build.yml

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ jobs:
4040
run: |
4141
python -m pip install --upgrade pip wheel
4242
pip install -e .
43+
pip install -e .[test,websurfer]
4344
python -c "import autogen"
4445
pip install pytest mock
4546
- name: Test with pytest skipping openai tests

autogen/agentchat/contrib/web_surfer.py

+16-11
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,22 @@
1-
import json
21
import copy
32
import logging
43
import re
5-
from dataclasses import dataclass
6-
from typing import Dict, List, Optional, Union, Callable, Literal, Tuple
7-
from autogen import Agent, ConversableAgent, AssistantAgent, UserProxyAgent, GroupChatManager, GroupChat, OpenAIWrapper
8-
from autogen.browser_utils import SimpleTextBrowser
9-
from autogen.code_utils import content_str
104
from datetime import datetime
11-
from autogen.token_count_utils import count_token, get_max_token_limit
5+
from typing import Dict, List, Optional, Union, Callable, Literal, Tuple
6+
7+
from autogen import Agent, ConversableAgent, AssistantAgent, UserProxyAgent, OpenAIWrapper
8+
from autogen.browser_utils import SimpleTextBrowser, HeadlessChromeBrowser
129
from autogen.oai.openai_utils import filter_config
10+
from autogen.token_count_utils import count_token, get_max_token_limit
1311

1412
logger = logging.getLogger(__name__)
1513

1614

1715
class WebSurferAgent(ConversableAgent):
18-
"""(In preview) An agent that acts as a basic web surfer that can search the web and visit web pages."""
16+
"""(In preview) An agent that acts as a basic web surfer that can search the web and visit web pages.
17+
Defaults to a simple text-based browser.
18+
Can be configured to use a headless Chrome browser by providing a browser_config dictionary with the key "headless" set to True.
19+
"""
1920

2021
DEFAULT_PROMPT = (
2122
"You are a helpful AI assistant with access to a web browser (via the provided functions). In fact, YOU ARE THE ONLY MEMBER OF YOUR PARTY WITH ACCESS TO A WEB BROWSER, so please help out where you can by performing web searches, navigating pages, and reporting what you find. Today's date is "
@@ -84,7 +85,11 @@ def __init__(
8485
if browser_config is None:
8586
self.browser = SimpleTextBrowser()
8687
else:
87-
self.browser = SimpleTextBrowser(**browser_config)
88+
headless = browser_config.pop("headless", False)
89+
if headless:
90+
self.browser = HeadlessChromeBrowser(**browser_config)
91+
else:
92+
self.browser = SimpleTextBrowser(**browser_config)
8893

8994
# Create a copy of the llm_config for the inner monologue agents to use, and set them up with function calling
9095
if llm_config is None: # Nothing to copy
@@ -214,7 +219,7 @@ def _browser_state():
214219
current_page = self.browser.viewport_current_page
215220
total_pages = len(self.browser.viewport_pages)
216221

217-
header += f"Viewport position: Showing page {current_page+1} of {total_pages}.\n"
222+
header += f"Viewport position: Showing page {current_page + 1} of {total_pages}.\n"
218223
return (header, self.browser.viewport)
219224

220225
def _informational_search(query):
@@ -225,7 +230,7 @@ def _informational_search(query):
225230
def _navigational_search(query):
226231
self.browser.visit_page(f"bing: {query}")
227232

228-
# Extract the first linl
233+
# Extract the first link
229234
m = re.search(r"\[.*?\]\((http.*?)\)", self.browser.page_content)
230235
if m:
231236
self.browser.visit_page(m.group(1))

autogen/browser_utils/__init__.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from .simple_text_browser import SimpleTextBrowser
2+
from .headless_chrome_browser import HeadlessChromeBrowser
3+
4+
__all__ = (
5+
"SimpleTextBrowser",
6+
"HeadlessChromeBrowser",
7+
)
+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
from abc import ABC, abstractmethod
2+
from typing import Optional, Union, Dict
3+
4+
5+
class AbstractBrowser(ABC):
6+
"""An abstract class for a web browser."""
7+
8+
@abstractmethod
9+
def __init__(
10+
self,
11+
start_page: Optional[str] = "about:blank",
12+
viewport_size: Optional[int] = 1024 * 8,
13+
downloads_folder: Optional[Union[str, None]] = None,
14+
bing_api_key: Optional[Union[str, None]] = None,
15+
request_kwargs: Optional[Union[Dict, None]] = None,
16+
):
17+
pass
18+
19+
@property
20+
@abstractmethod
21+
def address(self) -> str:
22+
pass
23+
24+
@abstractmethod
25+
def set_address(self, uri_or_path):
26+
pass
27+
28+
@property
29+
@abstractmethod
30+
def viewport(self) -> str:
31+
pass
32+
33+
@property
34+
@abstractmethod
35+
def page_content(self) -> str:
36+
pass
37+
38+
@abstractmethod
39+
def page_down(self):
40+
pass
41+
42+
@abstractmethod
43+
def page_up(self):
44+
pass
45+
46+
@abstractmethod
47+
def visit_page(self, path_or_uri):
48+
pass
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import re
2+
3+
from bs4 import BeautifulSoup
4+
from selenium import webdriver
5+
from selenium.webdriver.chrome.options import Options
6+
from selenium.webdriver.common.by import By
7+
from typing import Optional, Union, Dict
8+
9+
from autogen.browser_utils.abstract_browser import AbstractBrowser
10+
11+
# Optional PDF support
12+
IS_PDF_CAPABLE = False
13+
try:
14+
import pdfminer
15+
import pdfminer.high_level
16+
17+
IS_PDF_CAPABLE = True
18+
except ModuleNotFoundError:
19+
pass
20+
21+
# Other optional dependencies
22+
try:
23+
import pathvalidate
24+
except ModuleNotFoundError:
25+
pass
26+
27+
28+
class HeadlessChromeBrowser(AbstractBrowser):
29+
"""(In preview) A Selenium powered headless Chrome browser. Suitable for Agentic use."""
30+
31+
def __init__(
32+
self,
33+
start_page: Optional[str] = "about:blank",
34+
viewport_size: Optional[int] = 1024 * 8,
35+
downloads_folder: Optional[Union[str, None]] = None,
36+
bing_api_key: Optional[Union[str, None]] = None,
37+
request_kwargs: Optional[Union[Dict, None]] = None,
38+
):
39+
self.start_page = start_page
40+
self.driver = None
41+
self.viewport_size = viewport_size # Applies only to the standard uri types
42+
self.downloads_folder = downloads_folder
43+
self.history = list()
44+
self.page_title = None
45+
self.viewport_current_page = 0
46+
self.viewport_pages = list()
47+
self.bing_api_key = bing_api_key
48+
self.request_kwargs = request_kwargs
49+
self._page_content = ""
50+
51+
self._start_browser()
52+
53+
def _start_browser(self):
54+
chrome_options = Options()
55+
chrome_options.add_argument("--headless")
56+
self.driver = webdriver.Chrome(options=chrome_options)
57+
self.driver.get(self.start_page)
58+
59+
@property
60+
def address(self) -> str:
61+
return self.driver.current_url
62+
63+
def set_address(self, uri_or_path):
64+
if uri_or_path.startswith("bing:"):
65+
self._bing_search(uri_or_path[len("bing:") :].strip())
66+
else:
67+
self.driver.get(uri_or_path)
68+
69+
@property
70+
def viewport(self) -> str:
71+
"""Return the content of the current viewport."""
72+
if not self.viewport_pages:
73+
return ""
74+
bounds = self.viewport_pages[self.viewport_current_page]
75+
return self._page_content[bounds[0] : bounds[1]]
76+
77+
@property
78+
def page_content(self) -> str:
79+
"""Return the full contents of the current page."""
80+
return self._page_content
81+
82+
def _set_page_content(self, content) -> str:
83+
"""Sets the text content of the current page."""
84+
self._page_content = content
85+
self._split_pages()
86+
if self.viewport_current_page >= len(self.viewport_pages):
87+
self.viewport_current_page = len(self.viewport_pages) - 1
88+
89+
def _split_pages(self):
90+
# Split only regular pages
91+
if not self.address.startswith("http:") and not self.address.startswith("https:"):
92+
return
93+
94+
# Handle empty pages
95+
if len(self._page_content) == 0:
96+
self.viewport_pages = [(0, 0)]
97+
return
98+
99+
# Break the viewport into pages
100+
self.viewport_pages = []
101+
start_idx = 0
102+
while start_idx < len(self._page_content):
103+
end_idx = min(start_idx + self.viewport_size, len(self._page_content))
104+
self.viewport_pages.append((start_idx, end_idx))
105+
start_idx = end_idx
106+
107+
def _process_html(self, html: str) -> str:
108+
"""Process the raw HTML content and return the processed text."""
109+
soup = BeautifulSoup(html, "html.parser")
110+
111+
# Remove javascript and style blocks
112+
for script in soup(["script", "style"]):
113+
script.extract()
114+
115+
# Convert to text
116+
text = soup.get_text()
117+
118+
# Remove excessive blank lines
119+
text = re.sub(r"\n{2,}", "\n\n", text).strip()
120+
121+
return text
122+
123+
def _bing_search(self, query):
124+
self.driver.get("https://www.bing.com")
125+
126+
search_bar = self.driver.find_element(By.NAME, "q")
127+
search_bar.clear()
128+
search_bar.send_keys(query)
129+
search_bar.submit()
130+
131+
def page_down(self):
132+
"""Move the viewport one page down."""
133+
if self.viewport_current_page < len(self.viewport_pages) - 1:
134+
self.viewport_current_page += 1
135+
136+
def page_up(self):
137+
"""Move the viewport one page up."""
138+
if self.viewport_current_page > 0:
139+
self.viewport_current_page -= 1
140+
141+
def visit_page(self, path_or_uri):
142+
"""Update the address, visit the page, and return the content of the viewport."""
143+
self.set_address(path_or_uri)
144+
html = self.driver.execute_script("return document.body.innerHTML;")
145+
self._set_page_content(self._process_html(html))
146+
return self.viewport

autogen/browser_utils.py renamed to autogen/browser_utils/simple_text_browser.py

+9-8
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
1-
import json
1+
import io
2+
import mimetypes
23
import os
3-
import requests
44
import re
5-
import markdownify
6-
import io
5+
from typing import Optional, Union, Dict
76
import uuid
8-
import mimetypes
97
from urllib.parse import urljoin, urlparse
8+
9+
import markdownify
10+
import requests
1011
from bs4 import BeautifulSoup
11-
from dataclasses import dataclass
12-
from typing import Dict, List, Optional, Union, Callable, Literal, Tuple
12+
13+
from autogen.browser_utils.abstract_browser import AbstractBrowser
1314

1415
# Optional PDF support
1516
IS_PDF_CAPABLE = False
@@ -28,7 +29,7 @@
2829
pass
2930

3031

31-
class SimpleTextBrowser:
32+
class SimpleTextBrowser(AbstractBrowser):
3233
"""(In preview) An extremely simple text-based web browser comparable to Lynx. Suitable for Agentic use."""
3334

3435
def __init__(

setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
"teachable": ["chromadb"],
5353
"lmm": ["replicate", "pillow"],
5454
"graphs": ["networkx~=3.2.1", "matplotlib~=3.8.1"],
55-
"websurfer": ["beautifulsoup4", "markdownify", "pdfminer.six", "pathvalidate"],
55+
"websurfer": ["beautifulsoup4", "markdownify", "pdfminer.six", "pathvalidate", "selenium"],
5656
"redis": ["redis"],
5757
},
5858
classifiers=[

0 commit comments

Comments
 (0)