Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/webcanvas integration #294

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
version 1.1
manjaro1124 authored and han032206 committed Dec 20, 2024
commit f6775e749e297736e5ebbce43e87eaed73f44175
124 changes: 91 additions & 33 deletions browsergym/core/src/browsergym/core/env.py
Original file line number Diff line number Diff line change
@@ -270,7 +270,7 @@ def override_property(task, env, property):

self.context.expose_binding(
"handleEvent", lambda source,
selector, event_type,element_text: self._handle_event(selector, event_type,element_text)
selector, event_type, element_text: self._handle_event(selector, event_type, element_text)
)

self.context.add_init_script(
@@ -401,11 +401,13 @@ def report_infeasible_instructions(reason: str):

if hasattr(self.task, 'webcanvas'):
logger.debug(f"Initiating webcanvas task validation")
# extract reward, done, user_message, info (task-specific)
self.events = self.task.events
self._event_listener(
[event["selector"] for event in self.events if event and event["selector"] and event["status"] == False])

selectors = [event["selector"]
for event in self.events if event and event["selector"] and event["status"] == False]
element_value = [event["reference_value"]
for event in self.events if event and event["reference_value"] and event["status"] == False]
self._event_listener(selectors, element_value)

# reward, done, user_message, task_info = self.task.validate(
# self.page, self.chat.messages, action)
# logger.info(f"WebCanvas task validation result:\n{
@@ -459,7 +461,7 @@ def report_infeasible_instructions(reason: str):
# extract reward, done, user_message, info (task-specific)
reward, done, user_message, task_info = self._task_validate()
logger.info(f"WebCanvas task validation result:\n{
self.task.evaluate_result}")
self.task.evaluate_result}")
info["task_info"] = task_info
logger.debug(f"Task validation done")

@@ -485,8 +487,8 @@ def _task_validate(self):
prev_page_history = self.page_history.copy()
# call validate
reward, done, user_message, info = self.task.validate(
self.page, self.chat.messages,self.last_action)
# info["webcanvas_result"] = self.task.evaluate_result
self.page, self.chat.messages, self.last_action)
# info["webcanvas_result"] = self.task.evaluate_result
# safety fix, in case validate() did mess up the active page and/or page history
if prev_active_page != self.page or prev_page_history != self.page_history:
logger.debug(
@@ -617,34 +619,87 @@ def _get_obs(self):

return obs

def _event_listener(self, selectors):
# def _event_listener(self, selectors):
# """
# Add a universal event listener to specified selectors to capture various event types
# :param page: Current page object
# :param selectors: List of selectors to listen to
# """
# self.page.evaluate(
# """
# ({selectors}) => {
# selectors.forEach((selector) => {
# const element = document.querySelector(selector);
# if (element) {
# const allEvents = [
# 'click', 'input', 'change', 'keydown', 'keyup',
# 'mouseover', 'mouseout', 'mousedown', 'mouseup', 'focus', 'blur'
# ];
# allEvents.forEach((eventType) => {
# element.addEventListener(eventType, (event) => {
# const elementText = event.target.textContent || null;
# window.handleEvent(selector, eventType, elementText);
# }, true); // 'true' indicates capture phase
# });
# }
# });
# }
# """,
# {"selectors": selectors}
# )

def _event_listener(self, selectors: list = [], target_values: list = []):
"""
Add a universal event listener to specified selectors to capture various event types
Add event listeners to either specified selectors or globally across all elements.
:param page: Current page object
:param selectors: List of selectors to listen to
:param selectors: Optional list of selectors to listen to. If None, will apply globally.
:param target_values: Optional list of text content values to filter events by, used for global listening.
"""
self.page.evaluate(
"""
({selectors}) => {
selectors.forEach((selector) => {
const element = document.querySelector(selector);
if (element) {
const allEvents = [
'click', 'input', 'change', 'keydown', 'keyup',
'mouseover', 'mouseout', 'mousedown', 'mouseup', 'focus', 'blur'
];
allEvents.forEach((eventType) => {
element.addEventListener(eventType, (event) => {
const elementText = event.target.textContent || null;
window.handleEvent(selector, eventType, elementText);
}, true); // 'true' indicates capture phase
});
}
});
}
""",
{"selectors": selectors}
)
if selectors:
# Specific selectors case
self.page.evaluate(
"""
({selectors}) => {
selectors.forEach((selector) => {
const element = document.querySelector(selector);
if (element) {
const allEvents = [
'click', 'input', 'change', 'keydown', 'keyup',
'mouseover', 'mouseout', 'mousedown', 'mouseup', 'focus', 'blur'
];
allEvents.forEach((eventType) => {
element.addEventListener(eventType, (event) => {
const elementText = event.target.textContent || '';
window.handleEvent(selector, eventType, elementText);
}, true); // 'true' indicates capture phase
});
}
});
}
""",
{"selectors": selectors}
)
elif target_values:

self.page.evaluate(
"""
(targetValues) => {
const allEvents = [
'click', 'input', 'change', 'keydown', 'keyup',
'mouseover', 'mouseout', 'mousedown', 'mouseup', 'focus', 'blur'
];
allEvents.forEach((eventType) => {
document.addEventListener(eventType, (event) => {
const elementText = event.target.textContent || '';
if (targetValues.includes(elementText)) {
window.handleEvent(null, eventType, elementText); // No selector in this case
}
}, true); // 'true' indicates capture phase
});
}
""",
target_values
)

def _handle_event(self, selector, event_type, element_text=None):
logger.debug(f"Element with selector '{selector}' triggered '{
@@ -653,4 +708,7 @@ def _handle_event(self, selector, event_type, element_text=None):
if event and event["selector"] == selector:
self.events[idx]["status"] = True
self.events[idx]["target_value"] = element_text if element_text else ""
elif event and event["reference_value"] == element_text:
self.events[idx]["status"] = True
self.events[idx]["target_value"] = element_text if element_text else ""
self.task.update_events(self.events)
2 changes: 1 addition & 1 deletion browsergym/webcanvas/src/browsergym/webcanvas/__init__.py
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@

ALL_WEBCANVAS_TASK_IDS = []

for task_id in config.TASK_IDS:
for task_id in config.TASK_TRAIN_IDS:
gym_id = f"webcanvas.mind2web-live.{task_id}"
register_task(
gym_id,
3 changes: 2 additions & 1 deletion browsergym/webcanvas/src/browsergym/webcanvas/config.py
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
TASK_IDS = range(104)
TASK_TEST_IDS = range(104)
TASK_TRAIN_IDS = range(130)
Original file line number Diff line number Diff line change
@@ -63,7 +63,7 @@
{
"match_function_name": "element_value_exactly_match",
"content": {
"path": "#main > div:nth-child(3) > section > div > p:nth-child(3) > a",

"reference_answer": "View Full Menu",
"netloc": "amctheatres",
"url": "https://www.amctheatres.com/food-and-drink/dine-in/explore-menu"
16 changes: 8 additions & 8 deletions browsergym/webcanvas/src/browsergym/webcanvas/step_score.py
Original file line number Diff line number Diff line change
@@ -181,20 +181,20 @@ def element_value_exact_match(input_answer, reference_answer, input_netloc, refe

@ staticmethod
def element_value_include_match(input_answer, reference_answer, input_netloc, reference_netloc):
if reference_netloc != input_netloc:
# print("reference_netloc:", reference_netloc,
# "input_netloc:", input_netloc)
return 0
# if reference_netloc != input_netloc:
# # print("reference_netloc:", reference_netloc,
# # "input_netloc:", input_netloc)
# return 0
result_score = MatchFunction.include_match(
input_answer, reference_answer)
return result_score

@ staticmethod
def element_value_semantic_match(input_answer, semantic_method, input_netloc, reference_netloc=0):
if reference_netloc != input_netloc:
# print("reference_netloc:", reference_netloc,
# "input_netloc:", input_netloc)
return 0
# if reference_netloc != input_netloc:
# # print("reference_netloc:", reference_netloc,
# # "input_netloc:", input_netloc)
# return 0
if len(input_answer) == 0:
return 0
result_score = MatchFunction.semantic_match(
3 changes: 1 addition & 2 deletions browsergym/webcanvas/src/browsergym/webcanvas/task.py
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ def __init__(
# read the list of all WebCanvas task configs
import browsergym.webcanvas as wcs
all_configs_str = importlib.resources.files(wcs).joinpath(
"data/mind2web-train_130.json").read_text()
"data/mind2web-live-train_130.json").read_text()
all_task_configs = json.loads(all_configs_str)
all_task = WebCanvasInstance.read_task_configs(all_task_configs)
if task_id is not None and task_id < len(all_task):
@@ -150,7 +150,6 @@ def validate(
if self.task_finished:
done = True
break
print(self.task_events)
self.trace_info.append(step_action_info)
return reward, done, msg, info

21 changes: 14 additions & 7 deletions browsergym/webcanvas/src/browsergym/webcanvas/utils.py
Original file line number Diff line number Diff line change
@@ -127,6 +127,13 @@ def check_event_by_selector(events, selector):
return 1, event
return 0, None

def check_event_by_element_value(events, element_value):
for event in events:
if event and event["target_value"] == element_value:
if event["status"]:
return 1, event
return 0, None

step_score = 0
match_result = []
for evaluate in evaluate_steps:
@@ -159,10 +166,10 @@ def check_event_by_selector(events, selector):
else:
score = ElementEvaluator.element_value_exact_match(
event["target_value"], evaluate["reference_answer"], input_netloc, evaluate["netloc"])
print("score:",score)
# print("score:",score)
else:
score = ElementEvaluator.element_value_exact_match(
target_value, evaluate["reference_answer"], input_netloc, evaluate["netloc"])
score, _ = check_event_by_element_value(
task_events, evaluate["reference_answer"])

elif match_function == "element_value_included_match":
input_netloc = get_netloc(page.url)
@@ -175,8 +182,8 @@ def check_event_by_selector(events, selector):
score = ElementEvaluator.element_value_include_match(
event["target_value"], evaluate["reference_answer"], input_netloc, evaluate["netloc"])
else:
score = ElementEvaluator.element_value_include_match(
event["target_value"], evaluate["reference_answer"], input_netloc, evaluate["netloc"])
score, _ = check_event_by_element_value(
task_events, evaluate["reference_answer"])

elif match_function == "element_value_semantic_match":
input_netloc = get_netloc(page.url)
@@ -189,8 +196,8 @@ def check_event_by_selector(events, selector):
score = ElementEvaluator.element_value_semantic_match(
event["target_value"], evaluate["reference_answer"], input_netloc, evaluate["netloc"])
else:
score = ElementEvaluator.element_value_semantic_match(
event["target_value"], evaluate["reference_answer"], input_netloc, evaluate["netloc"])
score, _ = check_event_by_element_value(
task_events, evaluate["reference_answer"])

evaluate["score"] = max(evaluate["score"], score)
if evaluate["score"] >= 1:
98 changes: 56 additions & 42 deletions test3.py
Original file line number Diff line number Diff line change
@@ -1,47 +1,65 @@
from playwright.sync_api import sync_playwright, Playwright

# Define a callback function to handle event triggers


def handle_event(selector, event_type, element_text=None):
print(f"Element with selector '{selector}' triggered '{
event_type}' event, text content: {element_text}")

print(f"Element with selector '{selector}' triggered '{event_type}' event, text content: {element_text}")

def event_listener(page, selectors):
def add_event_listener(page, selectors=None, target_values=None):
"""
Add a universal event listener to specified selectors to capture various event types
Add event listeners to either specified selectors or globally across all elements.
:param page: Current page object
:param selectors: List of selectors to listen to
:param selectors: Optional list of selectors to listen to. If None, will apply globally.
:param target_values: Optional list of text content values to filter events by, used for global listening.
"""
page.evaluate(
"""
({selectors}) => {
selectors.forEach((selector) => {
const element = document.querySelector(selector);
if (element) {
const allEvents = [
'click', 'input', 'change', 'keydown', 'keyup',
'mouseover', 'mouseout', 'mousedown', 'mouseup', 'focus', 'blur'
];
allEvents.forEach((eventType) => {
element.addEventListener(eventType, (event) => {
const elementText = event.target.textContent || null;
window.handleEvent(selector, eventType, elementText);
}, true); // 'true' indicates capture phase
});
}
});
}
""",
{"selectors": selectors}
)
if selectors:
# Specific selectors case
page.evaluate(
"""
({selectors}) => {
selectors.forEach((selector) => {
const element = document.querySelector(selector);
if (element) {
const allEvents = [
'click', 'input', 'change', 'keydown', 'keyup',
'mouseover', 'mouseout', 'mousedown', 'mouseup', 'focus', 'blur'
];
allEvents.forEach((eventType) => {
element.addEventListener(eventType, (event) => {
const elementText = event.target.textContent || '';
window.handleEvent(selector, eventType, elementText);
}, true); // 'true' indicates capture phase
});
}
});
}
""",
{"selectors": selectors}
)
elif target_values:
# Global listener with multiple target_values filtering
page.evaluate(
"""
(targetValues) => {
const allEvents = [
'click', 'input', 'change', 'keydown', 'keyup',
'mouseover', 'mouseout', 'mousedown', 'mouseup', 'focus', 'blur'
];
allEvents.forEach((eventType) => {
document.addEventListener(eventType, (event) => {
const elementText = event.target.textContent || '';
if (targetValues.includes(elementText)) {
window.handleEvent(null, eventType, elementText); // No selector in this case
}
}, true); // 'true' indicates capture phase
});
}
""",
target_values
)
return page


def run(playwright: Playwright):
chromium = playwright.chromium
# Set to True to enable headless mode
browser = chromium.launch(headless=False)
context = browser.new_context()
page = context.new_page()
@@ -53,27 +71,23 @@ def run(playwright: Playwright):
page.wait_for_load_state('load')

# Expose the Python function to the page context
context.expose_binding("handleEvent", lambda source, selector, event_type,
element_text: handle_event(selector, event_type, element_text))
context.expose_binding("handleEvent", lambda source, selector, event_type, element_text: handle_event(selector, event_type, element_text))

# Call the function to add a universal event listener to specified selectors
page = event_listener(
# Add event listeners with specific selectors and multiple target values
add_event_listener(
page,
[
"#main > div:nth-child(3) > section > div > p:nth-child(3) > a"
] # Add selectors you want to listen to
selectors=[], # Add specific selectors
target_values=["View Full Menu", "Movies"] # List of target values for global monitoring
)

# Simulate clicking on an element
locator = page.locator(
"#main > div:nth-child(3) > section > div > p:nth-child(3) > a")
locator = page.locator("#main > div:nth-child(3) > section > div > p:nth-child(3) > a")
locator.click()

page.wait_for_timeout(3000)

# Close the browser
browser.close()


with sync_playwright() as playwright:
run(playwright)