diff --git a/envs/browsergym_env/server/browsergym_environment.py b/envs/browsergym_env/server/browsergym_environment.py index a66734994..9b2bc0019 100644 --- a/envs/browsergym_env/server/browsergym_environment.py +++ b/envs/browsergym_env/server/browsergym_environment.py @@ -6,17 +6,20 @@ - WebArena: Realistic evaluation with 812 complex tasks - VisualWebArena: Visual web navigation tasks - WorkArena: Enterprise task automation +- Custom: User-defined tasks with custom HTML and reward logic """ import importlib import logging -from typing import Any, Dict, Optional +import os +import sys +from typing import Any, Dict, Optional, TYPE_CHECKING from uuid import uuid4 import gymnasium as gym from openenv.core.env_server.interfaces import Environment -from browsergym_env.models import ( +from envs.browsergym_env.models import ( BrowserGymAction, BrowserGymObservation, BrowserGymState, @@ -24,6 +27,38 @@ logger = logging.getLogger(__name__) +# Add the server directory to sys.path to allow custom module imports +_SERVER_DIR = os.path.dirname(os.path.abspath(__file__)) +if _SERVER_DIR not in sys.path: + sys.path.insert(0, _SERVER_DIR) # noqa: E402 + +# Import custom models for custom benchmark +if TYPE_CHECKING: + from custom.custom_models import ( + CustomGymAction, + CustomGymObservation, + CustomGymState, + ) + +try: + from custom.custom_models import ( + CustomGymAction as _CustomGymAction, + CustomGymObservation as _CustomGymObservation, + CustomGymState as _CustomGymState, + ) + + CUSTOM_AVAILABLE = True + CustomGymAction = _CustomGymAction + CustomGymObservation = _CustomGymObservation + CustomGymState = _CustomGymState + _CUSTOM_IMPORT_ERROR = None +except ImportError as e: + CUSTOM_AVAILABLE = False + CustomGymAction = None # type: ignore + CustomGymObservation = None # type: ignore + CustomGymState = None # type: ignore + _CUSTOM_IMPORT_ERROR = str(e) + def _get_axtree_txt(obs: Dict[str, Any]) -> str: """Extract accessibility tree text from BrowserGym observation. @@ -124,59 +159,103 @@ def __init__( self.timeout = timeout self.gym_kwargs = dict(gym_kwargs) - # Build environment ID - if task_name: - self.env_id = f"browsergym/{benchmark}.{task_name}" + # Check if this is a custom benchmark + self.is_custom = benchmark == "custom" + + if self.is_custom: + # Custom benchmark handling + if not CUSTOM_AVAILABLE: + raise ValueError( + f"Custom benchmark requested but custom module import failed: {_CUSTOM_IMPORT_ERROR}" + ) + + if not task_name: + raise ValueError("task_name is required for custom benchmark") + + # Import and instantiate the custom task + try: + from custom.custom_tasks import get_custom_task + + self.custom_env = get_custom_task( + task_name, + headless=headless, + viewport_width=viewport_width, + viewport_height=viewport_height, + timeout=timeout, + ) + except ImportError as e: + raise ValueError( + f"Failed to import custom task '{task_name}': {e}\n" + f"Make sure the task is registered in custom/custom_tasks.py" + ) from e + + self.gym_env = None + self.env_id = f"custom/{task_name}" + + # Use CustomGymState for custom benchmarks + self._state = CustomGymState( + episode_id=str(uuid4()), + step_count=0, + benchmark="custom", + task_name=task_name, + ) else: - self.env_id = f"browsergym/{benchmark}" - - # force import the benchmark module - benchmark_modules = { - "miniwob": "browsergym.miniwob", - "webarena": "browsergym.webarena", - "visualwebarena": "browsergym.visualwebarena", - "workarena": "browsergym.workarena", - } - module_path = benchmark_modules.get(benchmark) - try: - if module_path: - importlib.import_module(module_path) + # Original BrowserGym benchmark handling + # Build environment ID + if task_name: + self.env_id = f"browsergym/{benchmark}.{task_name}" else: - importlib.import_module("browsergym") - except ModuleNotFoundError as import_error: - message = ( - "Failed to import BrowserGym benchmark " - f"'{benchmark}': {import_error}\n" - "Install the matching browsergym package " - f"(e.g., browsergym-{benchmark})." - ) - raise ValueError(message) from import_error - - # Create the BrowserGym environment - try: - self.gym_env = gym.make( - self.env_id, - headless=headless, - viewport={"width": viewport_width, "height": viewport_height}, - timeout=timeout, - **self.gym_kwargs, - ) - except Exception as e: # noqa: BLE001 - gym.make - message = ( - "Failed to create BrowserGym environment " - f"'{self.env_id}': {e}\n" - "Make sure the benchmark package is installed " - f"(e.g., pip install browsergym-{benchmark})." + self.env_id = f"browsergym/{benchmark}" + + # force import the benchmark module + benchmark_modules = { + "miniwob": "browsergym.miniwob", + "webarena": "browsergym.webarena", + "visualwebarena": "browsergym.visualwebarena", + "workarena": "browsergym.workarena", + } + module_path = benchmark_modules.get(benchmark) + try: + if module_path: + importlib.import_module(module_path) + else: + importlib.import_module("browsergym") + except ModuleNotFoundError as import_error: + message = ( + "Failed to import BrowserGym benchmark " + f"'{benchmark}': {import_error}\n" + "Install the matching browsergym package " + f"(e.g., browsergym-{benchmark})." + ) + raise ValueError(message) from import_error + + # Create the BrowserGym environment + try: + self.gym_env = gym.make( + self.env_id, + headless=headless, + viewport={"width": viewport_width, "height": viewport_height}, + timeout=timeout, + **self.gym_kwargs, + ) + except Exception as e: # noqa: BLE001 - gym.make + message = ( + "Failed to create BrowserGym environment " + f"'{self.env_id}': {e}\n" + "Make sure the benchmark package is installed " + f"(e.g., pip install browsergym-{benchmark})." + ) + raise ValueError(message) from e + + # State tracking + self._state = BrowserGymState( + episode_id=str(uuid4()), + step_count=0, + benchmark=benchmark, + task_name=task_name or "", ) - raise ValueError(message) from e - # State tracking - self._state = BrowserGymState( - episode_id=str(uuid4()), - step_count=0, - benchmark=benchmark, - task_name=task_name or "", - ) + self.custom_env = None self._last_obs: Optional[Dict[str, Any]] = None self._last_info: Optional[Dict[str, Any]] = None @@ -195,6 +274,13 @@ def reset( Returns: Initial observation for the task """ + if self.is_custom: + # Handle custom environment reset + obs = self.custom_env.reset(seed=seed) + self._state = self.custom_env.state + # Convert CustomGymObservation to BrowserGymObservation + return self._convert_custom_observation(obs) + # Generate new episode ID self._state = BrowserGymState( episode_id=str(uuid4()), @@ -240,6 +326,13 @@ def step(self, action: BrowserGymAction) -> BrowserGymObservation: Returns: Observation after executing the action """ + if self.is_custom: + # Handle custom environment step + custom_action = CustomGymAction(action_str=action.action_str) + obs = self.custom_env.step(custom_action) + self._state = self.custom_env.state + return self._convert_custom_observation(obs) + self._state.step_count += 1 # Execute action in gym environment @@ -364,6 +457,28 @@ def _create_observation( metadata=browsergym_metadata, ) + def _convert_custom_observation(self, custom_obs: "CustomGymObservation") -> BrowserGymObservation: + """Convert CustomGymObservation to BrowserGymObservation. + + Args: + custom_obs: Observation from custom environment + + Returns: + BrowserGymObservation compatible with OpenEnv interface + """ + return BrowserGymObservation( + text=custom_obs.text, + url=custom_obs.url, + goal=custom_obs.goal, + axtree_txt=custom_obs.text, # Reuse text for compatibility + pruned_html="", + error=custom_obs.error, + last_action_error=custom_obs.last_action_error, + done=custom_obs.done, + reward=custom_obs.reward, + metadata=custom_obs.metadata, + ) + @property def state(self) -> BrowserGymState: """Get the current environment state.""" @@ -371,5 +486,7 @@ def state(self) -> BrowserGymState: def close(self) -> None: """Clean up environment resources.""" - if hasattr(self, "gym_env"): + if self.is_custom and self.custom_env: + self.custom_env.close() + elif hasattr(self, "gym_env") and self.gym_env: self.gym_env.close() diff --git a/envs/browsergym_env/server/custom/__init__.py b/envs/browsergym_env/server/custom/__init__.py new file mode 100644 index 000000000..885973c4f --- /dev/null +++ b/envs/browsergym_env/server/custom/__init__.py @@ -0,0 +1,9 @@ +"""Custom BrowserGym tasks module. + +This module provides custom task functionality for BrowserGym environments. +Custom tasks are registered in custom_tasks.py and can be used by setting +benchmark="custom" in BrowserGymEnvironment. +""" + +# The custom tasks are registered in custom_tasks.py +# No need to import anything here - imports happen when needed diff --git a/envs/browsergym_env/server/custom/custom_base.py b/envs/browsergym_env/server/custom/custom_base.py new file mode 100644 index 000000000..93d5a9068 --- /dev/null +++ b/envs/browsergym_env/server/custom/custom_base.py @@ -0,0 +1,337 @@ +"""Base custom environment for BrowserGym custom tasks. + +This module provides a base class for creating custom BrowserGym tasks that +are not part of the official benchmarks. It simulates the BrowserGym gym +environment interface using Playwright directly. +""" + +import asyncio +from abc import abstractmethod +from typing import Any, Dict, Optional +from uuid import uuid4 + +from playwright.async_api import async_playwright, Browser, Page, Playwright + +from .custom_models import ( + CustomGymAction, + CustomGymObservation, + CustomGymState, +) + + +class CustomBrowserGymEnvironment: + """Base class for custom BrowserGym environments. + + This class provides the basic Gym-like interface (reset, step, close) + but uses Playwright directly instead of going through BrowserGym's + registration system. + + To create a custom task: + 1. Subclass this class + 2. Implement _get_task_url() to return the starting URL + 3. Implement _extract_observation() to parse page state + 4. Implement _calculate_reward() to compute rewards + 5. Implement _check_done() to determine episode termination + """ + + def __init__( + self, + task_name: str, + headless: bool = True, + viewport_width: int = 1280, + viewport_height: int = 720, + timeout: float = 10000.0, + max_steps: int = 50, + **kwargs: Any, + ): + """Initialize the custom environment. + + Args: + task_name: Name of your custom task + headless: Whether to run browser in headless mode + viewport_width: Browser viewport width + viewport_height: Browser viewport height + timeout: Action timeout in milliseconds + max_steps: Maximum steps per episode + **kwargs: Additional custom parameters + """ + self.task_name = task_name + self.headless = headless + self.viewport_width = viewport_width + self.viewport_height = viewport_height + self.timeout = timeout + self.max_steps = max_steps + self.custom_params = kwargs + + # Playwright objects (initialized in reset) + self._playwright: Optional[Playwright] = None + self._browser: Optional[Browser] = None + self._page: Optional[Page] = None + self._event_loop: Optional[asyncio.AbstractEventLoop] = None + + # State tracking + self._state = CustomGymState( + episode_id=str(uuid4()), + step_count=0, + benchmark="custom", + task_name=task_name, + max_steps=max_steps, + ) + + @abstractmethod + def _get_task_url(self) -> str: + """Get the starting URL for this task. + + Returns: + URL to navigate to when resetting the environment + """ + pass + + @abstractmethod + def _get_goal_description(self) -> str: + """Get the goal/instruction for this task. + + Returns: + Human-readable description of the task goal + """ + pass + + @abstractmethod + async def _extract_observation(self, page: Page) -> Dict[str, Any]: + """Extract observation data from the current page state. + + Args: + page: Playwright Page object + + Returns: + Dictionary with observation data (text, axtree_txt, etc.) + """ + pass + + @abstractmethod + def _calculate_reward( + self, page_data: Dict[str, Any], action: str, error: Optional[str] = None + ) -> float: + """Calculate reward for the current step. + + Args: + page_data: Data extracted from _extract_observation + action: Action that was executed + error: Error message if action failed + + Returns: + Reward value + """ + pass + + @abstractmethod + def _check_done(self, page_data: Dict[str, Any]) -> bool: + """Check if the episode should terminate. + + Args: + page_data: Data extracted from _extract_observation + + Returns: + True if episode should end, False otherwise + """ + pass + + def _get_or_create_event_loop(self) -> asyncio.AbstractEventLoop: + """Get or create an event loop for async operations.""" + try: + loop = asyncio.get_event_loop() + if loop.is_closed(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop + + async def _async_reset(self, seed: Optional[int] = None) -> CustomGymObservation: + """Async implementation of reset.""" + # Generate new episode ID + self._state = CustomGymState( + episode_id=str(uuid4()), + step_count=0, + benchmark="custom", + task_name=self.task_name, + max_steps=self.max_steps, + ) + + # Initialize Playwright if needed + if self._playwright is None: + self._playwright = await async_playwright().start() + self._browser = await self._playwright.chromium.launch( + headless=self.headless + ) + + # Create new page + if self._page: + await self._page.close() + + self._page = await self._browser.new_page( + viewport={ + "width": self.viewport_width, + "height": self.viewport_height, + } + ) + + # Set timeout + self._page.set_default_timeout(self.timeout) + + # Navigate to task URL + task_url = self._get_task_url() + await self._page.goto(task_url) + + # Extract initial observation + page_data = await self._extract_observation(self._page) + goal = self._get_goal_description() + + self._state.current_url = self._page.url + self._state.goal = goal + + return CustomGymObservation( + text=page_data.get("text", ""), + url=self._page.url, + screenshot=page_data.get("screenshot"), + goal=goal, + axtree_txt=page_data.get("axtree_txt", ""), + pruned_html=page_data.get("pruned_html", ""), + error="", + last_action_error=False, + done=False, + reward=0.0, + custom_data=page_data.get("custom_data"), + ) + + async def _async_step(self, action_str: str) -> CustomGymObservation: + """Async implementation of step.""" + self._state.step_count += 1 + + error_msg = "" + last_action_error = False + + try: + # Execute the action + # BrowserGym actions are Python-like function calls + # We need to parse and execute them + await self._execute_action(action_str) + + except Exception as e: + error_msg = str(e) + last_action_error = True + + # Extract observation + page_data = await self._extract_observation(self._page) + + # Calculate reward + reward = self._calculate_reward(page_data, action_str, error_msg) + self._state.cum_reward += reward + + # Check if done + done = self._check_done(page_data) or self._state.step_count >= self.max_steps + + # Update state + self._state.current_url = self._page.url + + return CustomGymObservation( + text=page_data.get("text", ""), + url=self._page.url, + screenshot=page_data.get("screenshot"), + goal=self._state.goal, + axtree_txt=page_data.get("axtree_txt", ""), + pruned_html=page_data.get("pruned_html", ""), + error=error_msg, + last_action_error=last_action_error, + done=done, + reward=reward, + custom_data=page_data.get("custom_data"), + ) + + async def _execute_action(self, action_str: str) -> None: + """Execute a BrowserGym-style action string. + + Args: + action_str: Action string like "click('button')" or "fill('input', 'text')" + """ + # Simple action parser - you can make this more sophisticated + action_str = action_str.strip() + + if action_str.startswith("click("): + # Extract selector from click('selector') + selector = action_str[6:-1].strip("'\"") + await self._page.click(selector) + + elif action_str.startswith("fill("): + # Extract selector and text from fill('selector', 'text') + parts = action_str[5:-1].split(",", 1) + selector = parts[0].strip().strip("'\"") + text = parts[1].strip().strip("'\"") if len(parts) > 1 else "" + await self._page.fill(selector, text) + + elif action_str.startswith("goto("): + # Extract URL from goto('url') + url = action_str[5:-1].strip("'\"") + await self._page.goto(url) + + elif action_str.startswith("press("): + # Extract key from press('key') + key = action_str[6:-1].strip("'\"") + await self._page.keyboard.press(key) + + elif action_str.startswith("scroll("): + # Extract direction from scroll('direction') + direction = action_str[7:-1].strip("'\"") + if direction == "down": + await self._page.mouse.wheel(0, 500) + elif direction == "up": + await self._page.mouse.wheel(0, -500) + + else: + # Try to execute as JavaScript if not recognized + await self._page.evaluate(action_str) + + def reset(self, seed: Optional[int] = None) -> CustomGymObservation: + """Reset the environment. + + Args: + seed: Random seed for reproducibility + + Returns: + Initial observation + """ + loop = self._get_or_create_event_loop() + return loop.run_until_complete(self._async_reset(seed)) + + def step(self, action: CustomGymAction) -> CustomGymObservation: + """Execute an action. + + Args: + action: Action to execute + + Returns: + Observation after executing the action + """ + loop = self._get_or_create_event_loop() + return loop.run_until_complete(self._async_step(action.action_str)) + + @property + def state(self) -> CustomGymState: + """Get the current environment state.""" + return self._state + + def close(self) -> None: + """Clean up environment resources.""" + + async def _async_close(): + if self._page: + await self._page.close() + if self._browser: + await self._browser.close() + if self._playwright: + await self._playwright.stop() + + if self._playwright: + loop = self._get_or_create_event_loop() + loop.run_until_complete(_async_close()) diff --git a/envs/browsergym_env/server/custom/custom_models.py b/envs/browsergym_env/server/custom/custom_models.py new file mode 100644 index 000000000..3c07be978 --- /dev/null +++ b/envs/browsergym_env/server/custom/custom_models.py @@ -0,0 +1,77 @@ +"""Data models for custom BrowserGym tasks. + +These models are used specifically for custom tasks that are not part of the +official BrowserGym benchmarks (miniwob, webarena, visualwebarena, workarena). +""" + +from typing import List, Optional, Dict, Any +from pydantic import Field + +from openenv.core.env_server.types import Action, Observation, State + + +class CustomGymAction(Action): + """Action to be executed in a custom BrowserGym environment. + + Custom actions support the same BrowserGym action format but may include + additional custom fields specific to your task. + + Example actions: + - "click('Submit button')" + - "fill('username', 'john@example.com')" + - "goto('https://example.com')" + - "scroll(down)" + - "send_keys('Enter')" + """ + + action_str: str = Field(..., description="Natural language action string") + + +class CustomGymObservation(Observation): + """Observation returned from a custom BrowserGym environment. + + Contains multiple observation modalities including text (accessibility tree + or DOM), visual (screenshot), and page metadata, plus custom fields. + """ + + text: str = Field(default="", description="Text representation of the page") + url: str = Field(default="", description="Current URL of the page") + screenshot: Optional[List[List[List[int]]]] = Field( + default=None, description="Screenshot as array [height, width, channels]" + ) + goal: str = Field(default="", description="Task goal/instruction") + axtree_txt: str = Field(default="", description="Full accessibility tree as text") + pruned_html: str = Field(default="", description="Pruned HTML content") + error: str = Field(default="", description="Error message if action failed") + last_action_error: bool = Field( + default=False, description="Whether last action resulted in error" + ) + custom_data: Optional[Dict[str, Any]] = Field( + default=None, description="Optional custom task-specific data" + ) + + +class CustomGymState(State): + """State of a custom BrowserGym environment. + + Tracks the current task and progress through an episode, plus custom state fields. + """ + + benchmark: str = Field( + default="custom", description="Benchmark name (always 'custom')" + ) + task_name: str = Field(default="", description="Specific custom task name") + task_id: Optional[str] = Field( + default=None, description="Task ID for custom task tracking" + ) + goal: str = Field(default="", description="Task goal/instruction") + current_url: str = Field(default="", description="Current URL of the active page") + max_steps: Optional[int] = Field( + default=None, description="Maximum steps allowed for this task" + ) + cum_reward: float = Field( + default=0.0, description="Cumulative reward for the current episode" + ) + custom_state: Optional[Dict[str, Any]] = Field( + default=None, description="Optional custom state data" + ) diff --git a/envs/browsergym_env/server/custom/custom_tasks.py b/envs/browsergym_env/server/custom/custom_tasks.py new file mode 100644 index 000000000..1a2f1adcb --- /dev/null +++ b/envs/browsergym_env/server/custom/custom_tasks.py @@ -0,0 +1,302 @@ +"""Registry for custom BrowserGym tasks. + +This module provides a central place to register and retrieve custom tasks. +Add your custom tasks here to make them available through the BrowserGym environment. +""" + +import os +from typing import Any, Dict + +from .custom_base import CustomBrowserGymEnvironment + + +# Registry of custom tasks +_CUSTOM_TASKS: Dict[str, type] = {} + + +def register_custom_task(name: str, task_class: type) -> None: + """Register a custom task. + + Args: + name: Task name (e.g., 'copy-paste', 'data-entry') + task_class: Class that extends CustomBrowserGymEnvironment + """ + if not issubclass(task_class, CustomBrowserGymEnvironment): + raise ValueError( + f"Task class must extend CustomBrowserGymEnvironment, got {task_class}" + ) + _CUSTOM_TASKS[name] = task_class + + +def get_custom_task(task_name: str, **kwargs: Any) -> CustomBrowserGymEnvironment: + """Get a custom task instance. + + Args: + task_name: Name of the task to retrieve + **kwargs: Arguments to pass to the task constructor + + Returns: + Instance of the custom task + + Raises: + ValueError: If task is not registered + """ + if task_name not in _CUSTOM_TASKS: + available = ", ".join(_CUSTOM_TASKS.keys()) or "none" + raise ValueError( + f"Custom task '{task_name}' not found. " + f"Available tasks: {available}. " + f"Register your task using register_custom_task()." + ) + + task_class = _CUSTOM_TASKS[task_name] + return task_class(task_name=task_name, **kwargs) + + +def list_custom_tasks() -> list[str]: + """List all registered custom tasks. + + Returns: + List of task names + """ + return list(_CUSTOM_TASKS.keys()) + + +# ============================================================================ +# Copy-Paste in a single page HTML task +# ============================================================================ + + +class CopyPasteTask(CustomBrowserGymEnvironment): + """Copy text from one field and paste into another.""" + + def _get_task_url(self) -> str: + """Get the URL for the copy-paste task.""" + task_html = os.path.join(os.path.dirname(__file__), "tasks", "copy-paste.html") + return f"file://{task_html}" + + def _get_goal_description(self) -> str: + """Get the goal description.""" + return "Copy the text from the source field and paste it into the target field, then click Submit." + + async def _extract_observation(self, page) -> dict: + """Extract observation from the page.""" + # Get the accessibility tree or HTML + try: + # Try to get the page content + content = await page.content() + + # Get the current values of source and target fields + source_value = await page.evaluate( + "document.querySelector('#source-text')?.value || ''" + ) + target_value = await page.evaluate( + "document.querySelector('#target-text')?.value || ''" + ) + + # Get success message if visible + success_msg = await page.evaluate( + "document.querySelector('#success-message')?.textContent || ''" + ) + + return { + "text": content, + "pruned_html": content[:1000], # Truncate for observation + "custom_data": { + "source_value": source_value, + "target_value": target_value, + "success_message": success_msg, + }, + } + except Exception as e: + return { + "text": f"Error extracting observation: {e}", + "custom_data": {"error": str(e)}, + } + + def _calculate_reward( + self, page_data: dict, action: str, error: str | None = None + ) -> float: + """Calculate reward based on page state.""" + if error: + return -0.1 # Small penalty for errors + + custom_data = page_data.get("custom_data", {}) + + # Check if task is completed successfully + if "Success!" in custom_data.get("success_message", ""): + return 1.0 + + # Partial reward if text is copied correctly + source = custom_data.get("source_value", "") + target = custom_data.get("target_value", "") + + if source and target and source == target: + return 0.5 + + return 0.0 + + def _check_done(self, page_data: dict) -> bool: + """Check if the task is complete.""" + custom_data = page_data.get("custom_data", {}) + # Task is done if success message is shown + return "Success!" in custom_data.get("success_message", "") + + +# Register the example task +register_custom_task("copy-paste", CopyPasteTask) + + +# ============================================================================ +# Multi-Tab Copy-Paste Task +# ============================================================================ + + +class CopyPasteMultiTabTask(CustomBrowserGymEnvironment): + """Copy text from one tab and paste it into another tab. + + This task demonstrates handling multiple browser tabs/pages. + The agent needs to: + 1. Copy text from the source page (tab 1) + 2. Navigate/switch to the target page (tab 2) + 3. Paste the text into the target field + 4. Submit the form + """ + + def _get_task_url(self) -> str: + """Get the URL for the first tab (source page).""" + + task_html = os.path.join( + os.path.dirname(__file__), "tasks", "copy-paste-source.html" + ) + return f"file://{task_html}" + + def _get_goal_description(self) -> str: + """Get the goal description.""" + return ( + "Copy the text from the source page, then navigate to the target page " + "(click 'Open Target Page' button), paste the text into the input field, " + "and click Submit." + ) + + async def _extract_observation(self, page) -> dict: + """Extract observation from the current page.""" + try: + content = await page.content() + current_url = page.url + + # Determine which page we're on + if "source" in current_url: + # On source page + source_value = await page.evaluate( + "document.querySelector('#source-text')?.textContent || ''" + ) + + return { + "text": content, + "pruned_html": content[:1000], + "custom_data": { + "current_page": "source", + "source_value": source_value, + "task_step": "copy_from_source", + }, + } + + elif "target" in current_url: + # On target page + target_value = await page.evaluate( + "document.querySelector('#target-text')?.value || ''" + ) + success_msg = await page.evaluate( + "document.querySelector('#success-message')?.textContent || ''" + ) + + return { + "text": content, + "pruned_html": content[:1000], + "custom_data": { + "current_page": "target", + "target_value": target_value, + "success_message": success_msg, + "task_step": "paste_to_target", + }, + } + + else: + # Unknown page + return { + "text": content, + "custom_data": { + "current_page": "unknown", + "error": "Not on source or target page", + }, + } + + except Exception as e: + return { + "text": f"Error extracting observation: {e}", + "custom_data": {"error": str(e)}, + } + + def _calculate_reward( + self, page_data: dict, action: str, error: str | None = None + ) -> float: + """Calculate reward based on page state and action.""" + if error: + return -0.1 + + custom_data = page_data.get("custom_data", {}) + current_page = custom_data.get("current_page", "") + + # Big reward for completing the task + if "Success!" in custom_data.get("success_message", ""): + return 1.0 + + # Small reward for successfully navigating to target page + if current_page == "target" and "goto" in action.lower(): + return 0.3 + + # Medium reward if text is pasted correctly in target + if current_page == "target": + target_value = custom_data.get("target_value", "") + # The expected text from source page + if target_value and "Hello from the source page!" in target_value: + return 0.6 + + return 0.0 + + def _check_done(self, page_data: dict) -> bool: + """Check if the task is complete.""" + custom_data = page_data.get("custom_data", {}) + return "Success!" in custom_data.get("success_message", "") + + +# Register the multi-tab task +register_custom_task("copy-paste-multitab", CopyPasteMultiTabTask) + + +# ============================================================================ +# Add your own custom tasks below by: +# 1. Creating a class that extends CustomBrowserGymEnvironment +# 2. Implementing the required methods +# 3. Registering it with register_custom_task() +# ============================================================================ + +# Example: +# class MyCustomTask(CustomBrowserGymEnvironment): +# def _get_task_url(self) -> str: +# return "https://my-task-url.com" +# +# def _get_goal_description(self) -> str: +# return "Do something amazing" +# +# async def _extract_observation(self, page) -> dict: +# return {"text": await page.content()} +# +# def _calculate_reward(self, page_data, action, error=None) -> float: +# return 1.0 if some_condition else 0.0 +# +# def _check_done(self, page_data) -> bool: +# return some_completion_check +# +# register_custom_task("my-task", MyCustomTask) diff --git a/envs/browsergym_env/server/custom/tasks/copy-paste-source.html b/envs/browsergym_env/server/custom/tasks/copy-paste-source.html new file mode 100644 index 000000000..137b65fe7 --- /dev/null +++ b/envs/browsergym_env/server/custom/tasks/copy-paste-source.html @@ -0,0 +1,37 @@ + + + + + + Source Page + + + +

Source Page

+

Text to copy:

+
Hello from the source page!
+ + + + + diff --git a/envs/browsergym_env/server/custom/tasks/copy-paste-target.html b/envs/browsergym_env/server/custom/tasks/copy-paste-target.html new file mode 100644 index 000000000..a20c5267a --- /dev/null +++ b/envs/browsergym_env/server/custom/tasks/copy-paste-target.html @@ -0,0 +1,94 @@ + + + + + + Target Page + + + + Back to Source Page + +

Target Page

+ +
+ + +
+ + + +
Success! You've completed the multi-tab copy-paste task correctly!
+
Error: The text doesn't match. Please copy the correct text from the source page.
+ + + + diff --git a/envs/browsergym_env/server/custom/tasks/copy-paste.html b/envs/browsergym_env/server/custom/tasks/copy-paste.html new file mode 100644 index 000000000..18eb2c3b6 --- /dev/null +++ b/envs/browsergym_env/server/custom/tasks/copy-paste.html @@ -0,0 +1,93 @@ + + + + + + Copy-Paste Task + + + +
+ + +
+ +
+ + +
+ + + +
Success! You've completed the task correctly.
+
Error: The text doesn't match. Please try again.
+ + + + \ No newline at end of file diff --git a/rfcs/005-generic-task-support.md b/rfcs/005-generic-task-support.md new file mode 100644 index 000000000..f720d214c --- /dev/null +++ b/rfcs/005-generic-task-support.md @@ -0,0 +1,450 @@ +# RFC: Generic Task Support + +**Status**: Draft +**Created**: 01/24/2026 +**Authors**: @atchudhansg +**RFC ID:** 005 + +## Summary +This RFC proposes a unified interface for defining, registering, and injecting tasks into OpenEnv environments. This allows environments to be decoupled from specific benchmarks or datasets, enabling dynamic task loading for training, evaluation, and custom scenarios. + +## Motivation +Currently, tasks are often hardcoded into the environment's `__init__` or tightly coupled with specific benchmark packages (e.g., BrowserGym benchmarks). This limits flexibility: +- Users cannot easily define custom tasks without modifying environment code. +- Switching tasks often requires re-initializing the environment. +- There is no standard way to define "what the agent should do" across different environment types (browser, coding, chat). + +As we move towards supporting training workflows and custom evaluations, we need a generic way to tell an environment "Here is a task, reset yourself to this state and evaluate the agent based on these criteria." + +### Use Cases +1. **Proprietary Task Evaluation**: Companies need to test agents on internal workflows without upstreaming tasks to public benchmarks +2. **Rapid Prototyping**: Researchers want to iterate on task designs without rebuilding Docker containers +3. **Curriculum Learning**: Training pipelines need to dynamically select tasks based on agent performance +4. **Domain-Specific Tasks**: Custom workflows (e.g., internal tools, enterprise software) that don't fit standard benchmarks + +## Architecture Overview + +### Component Diagram +``` +┌─────────────────────────────────────────────────────────────┐ +│ Client Side (Python/TypeScript) │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ env = BrowserGymEnv(...) │ │ +│ │ env.reset(task="custom/login-test") │ │ +│ └──────────────────────┬───────────────────────────────┘ │ +└─────────────────────────┼──────────────────────────────────┘ + │ HTTP/REST + │ +┌─────────────────────────▼──────────────────────────────────┐ +│ Docker Container (Server Side) │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ Environment Server (FastAPI) │ │ +│ │ ├─ BrowserGymEnvironment │ │ +│ │ │ └─ reset(task_id) → loads task from registry │ │ +│ │ └─ CustomBrowserGymEnvironment │ │ +│ │ ├─ _setup_page(html_content) │ │ +│ │ ├─ _calculate_reward(page_data, action) │ │ +│ │ └─ _check_done(state) │ │ +│ └────────────────────────────────────────────────────┘ │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ Task Registry │ │ +│ │ ├─ Built-in tasks (copy-paste, login-demo) │ │ +│ │ └─ Mounted tasks (/opt/openenv/custom_tasks/) │ │ +│ └────────────────────────────────────────────────────┘ │ +│ ┌────────────────────────────────────────────────────┐ │ +│ │ Task Definitions (Python classes) │ │ +│ │ - HTML templates │ │ +│ │ - Reward functions │ │ +│ │ - Termination conditions │ │ +│ └────────────────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────┘ +``` + +### Data Flow +1. **Client** calls `reset(task="custom/login-test")` +2. **Server** looks up task in registry +3. **Task class** provides HTML template and configuration +4. **Environment** sets up Playwright page with task's HTML +5. **Agent** receives initial observation +6. **Agent** sends actions (e.g., `click('#login-button')`) +7. **Task class** calculates rewards based on page state +8. **Server** returns observation with reward and done flag + +## Core Abstractions + +### Task Class Interface +All custom tasks must inherit from `CustomBrowserGymEnvironment` and implement: + +```python +class CustomBrowserGymEnvironment(ABC): + task_name: str = "base-task" + max_steps: int = 10 + + @abstractmethod + def _get_html_content(self) -> str: + """Return the HTML template for this task.""" + pass + + @abstractmethod + def _calculate_reward( + self, page_data: Dict[str, Any], action: str, error: Optional[str] = None + ) -> float: + """Calculate reward for the current step.""" + pass + + @abstractmethod + def _check_done(self, page_data: Dict[str, Any]) -> bool: + """Determine if the episode should terminate.""" + pass +``` + +### Task Registry +```python +# Global registry mapping task names to classes +_CUSTOM_TASKS: Dict[str, type] = {} + +def register_custom_task(name: str, task_class: type) -> None: + """Register a custom task.""" + _CUSTOM_TASKS[name] = task_class + +def get_custom_task(name: str) -> type: + """Retrieve a custom task class by name.""" + if name not in _CUSTOM_TASKS: + raise ValueError(f"Unknown custom task: {name}") + return _CUSTOM_TASKS[name] +``` + +### Environment Integration +```python +class BrowserGymEnvironment(Environment): + def __init__(self, benchmark: str = "miniwob", task_name: str = None, ...): + if benchmark == "custom": + # Load custom task + task_class = get_custom_task(task_name) + self._custom_env = task_class(...) + else: + # Use official BrowserGym benchmark + self._gym_env = gym.make(f"{benchmark}.{task_name}") +``` + +## Key Design Decisions + +### Decision 1: Python Classes vs. Declarative Config +**Chosen**: Python classes for task definitions + +**Rationale**: +- ✅ **Flexibility**: Complex reward logic, multi-step state tracking, dynamic HTML generation +- ✅ **Type Safety**: IDE support, runtime validation via type hints +- ✅ **Debugging**: Standard Python debugging tools work +- ✅ **Reusability**: Tasks can inherit from base classes, share utilities + +**Trade-offs**: +- ❌ **Barrier to Entry**: Requires Python knowledge vs. YAML/JSON config +- ❌ **Sandboxing**: Python code has full container privileges +- **Mitigation**: Provide task templates and examples; future work on validation + +**Alternative Considered**: YAML/JSON task definitions +```yaml +task: + name: login-test + html: tasks/login.html + reward: + type: element_present + selector: "#success-message" +``` +- Rejected because complex tasks (multi-step, stateful rewards) require code anyway + +### Decision 2: Server-Side Only (No Client-Side Injection) +**Chosen**: Tasks are always server-side within Docker containers + +**Rationale**: +- ✅ **Security**: Maintains environment boundary, prevents reward tampering +- ✅ **Consistency**: RFC 002 invariant "rewards inside environment" +- ✅ **Reproducibility**: Task code versioned with container image + +**Trade-offs**: +- ❌ **Iteration Speed**: Requires container rebuild or volume mount +- **Mitigation**: Document volume mount workflow for development + +### Decision 3: Task Registration via Imports (Not Dynamic Discovery) +**Chosen**: Tasks must be explicitly registered via `@register_custom_task` decorator + +**Rationale**: +- ✅ **Predictability**: Clear what tasks are available +- ✅ **Explicit**: No magic filesystem scanning +- ✅ **Control**: Environment chooses which tasks to enable + +**Trade-offs**: +- ❌ **Manual Step**: Developers must remember to register tasks +- **Mitigation**: Registration happens automatically via import in `__init__.py` + +### Decision 4: Parallel System (Not Replacing BrowserGym Integration) +**Chosen**: Custom tasks are a separate mode (`benchmark="custom"`), not a replacement + +**Rationale**: +- ✅ **Backward Compatibility**: Existing code using official benchmarks unaffected +- ✅ **Gradual Adoption**: Users can opt-in to custom tasks +- ✅ **Isolation**: Custom task bugs don't break official benchmark support + +**Trade-offs**: +- ❌ **Code Duplication**: Some overlap in action parsing, observation conversion +- **Future Work**: Extract common abstractions to shared base class + +## Migration Path + +### For Existing BrowserGym Users +No changes required. Continue using official benchmarks: +```python +env = BrowserGymEnv(benchmark="miniwob", task_name="click-test") +``` + +### For New Custom Task Users +1. **Define Task Class** in `server/custom/custom_tasks.py` +2. **Register Task** via `@register_custom_task("my-task")` decorator +3. **Use Custom Benchmark** mode: +```python +env = BrowserGymEnv(benchmark="custom", task_name="my-task") +``` + +### Coexistence +Both systems run side-by-side: +```python +# Official benchmark +env1 = BrowserGymEnv(benchmark="miniwob", task_name="click-test") + +# Custom task +env2 = BrowserGymEnv(benchmark="custom", task_name="copy-paste") +``` + +## Implementation Details + +### File Organization +``` +src/envs/browsergym_env/ +├── server/ +│ ├── app.py # FastAPI server +│ ├── browsergym_environment.py # Main environment class +│ └── custom/ +│ ├── __init__.py # Auto-registers tasks +│ ├── custom_base.py # Base class for custom tasks +│ ├── custom_models.py # Data models +│ ├── custom_tasks.py # Task registry + built-in tasks +│ └── tasks/ +│ ├── copy_paste.py # Example task +│ └── login_demo.py # Example task +``` + +### Task Development Workflow + +#### Option 1: Build-Time Inclusion (Production) +1. Add task to `server/custom/tasks/my_task.py` +2. Register in `server/custom/__init__.py` +3. Rebuild Docker image +4. Deploy + +#### Option 2: Volume Mount (Development) +1. Create task locally: `/local/dev/my_task.py` +2. Mount into container: +```bash +docker run -v /local/dev:/opt/openenv/custom_tasks \ + -e BROWSERGYM_BENCHMARK=custom \ + -e BROWSERGYM_TASK_NAME=my-task \ + browsergym-env:latest +``` +3. Environment imports tasks from mounted directory +4. Iterate without rebuilding + +### HTML Template Storage +Custom tasks define HTML inline or load from files: + +**Option A: Inline HTML** +```python +def _get_html_content(self) -> str: + return """ + + + + + """ +``` + +**Option B: External File** +```python +def _get_html_content(self) -> str: + template_path = Path(__file__).parent / "templates" / "login.html" + return template_path.read_text() +``` + +## Rejection Criteria + +When **NOT** to use custom tasks (use official BrowserGym benchmarks instead): + +### ❌ Task Belongs in Upstream Benchmark +If the task is: +- Generalizable across domains (not company-specific) +- Well-defined evaluation criteria +- Useful for the broader research community + +**Action**: Contribute to BrowserGym's MiniWoB++, WebArena, or WorkArena + +### ❌ Task Requires Complex Multi-Page Workflows +Custom tasks are designed for single-page or simple multi-tab scenarios. For complex navigation: +- Multiple domain interactions (e.g., booking.com → airline.com → hotel.com) +- Long chains of dependencies +- State persistence across sessions + +**Action**: Use WebArena or WorkArena benchmarks + +### ❌ Task Needs Real External Services +Custom tasks use static HTML or local servers. If you need: +- Live API integrations (Stripe, AWS, etc.) +- Real authentication flows +- Production systems + +**Action**: Use WorkArena or set up dedicated test environments + +### ✅ Good Use Cases for Custom Tasks +- **Internal tool testing**: Company-specific UIs not in public benchmarks +- **Controlled experiments**: A/B testing UI variations for RL research +- **Curriculum learning**: Progressively harder versions of a core task +- **Toy problems**: Simple environments for debugging agent logic + +## Proposal + +### 1. The Task Abstraction +We define a `Task` as a portable unit of work for an agent. A task definition should ideally be serializable (JSON/YAML) to allow for easy storage and transmission, though some complex tasks may require code. + +A `Task` generally consists of: +- **Metadata**: ID, name, description, tags. +- **Instruction**: The prompt or goal given to the agent (e.g., "Book a flight to NYC"). +- **Environment Configuration**: Initial state required for the task. + - *Browser*: Initial URL, cookies, local storage, HTML content. + - *Coding*: Initial file tree, git repo state. + - *Game*: Level configuration, seed. +- **Reward/Evaluation Logic**: How to measure success. + - This can be a reference to a pre-defined reward function (e.g., `reward_function: "exact_match"`). + - Or a custom script/snippet if the environment supports sandboxed execution of evaluation code. + +### 2. Environment Interface Update +We propose updating the `Environment.reset` method in the base `Environment` class to accept a task definition. + +```python +class Environment(ABC): + @abstractmethod + def reset(self, task: str | dict | None = None, **kwargs) -> Observation: + """ + Reset the environment. + + Args: + task: Can be: + - A string (Task ID) to look up in a registry. + - A dictionary/object defining the task configuration directly. + - None (default behavior, e.g., random task from loaded benchmark). + """ + pass +``` + +### 3. Task Registry +To manage tasks, we introduce a `TaskRegistry`. This allows users to register custom tasks and refer to them by ID. + +```python +# Conceptual usage +from openenv.core.tasks import registry + +registry.register( + task_id="custom/login-test", + env_type="browsergym", + config={ + "start_url": "http://localhost:8000/login", + "goal": "Login with user 'admin' and password '1234'", + "reward_fn": "check_url_contains('dashboard')" + } +) + +# In the environment +env.reset(task="custom/login-test") +``` + +### 4. Integration Examples + +#### BrowserGym (Based on recent PR) +The recent "Custom Task System" for BrowserGym fits this model perfectly. +- **Config**: Defines the HTML/JS or URL for the task. +- **Reset**: The `BrowserGymEnvironment` reads the task config and uses Playwright to set up the page. + +#### Coding Environment +- **Config**: A map of filenames to content, or a git commit hash. +- **Reset**: The environment cleans the workspace and writes the specified files. +- **Eval**: Runs a provided test command (e.g., `pytest test_task.py`). + +## Architectural Invariants + +This proposal adheres to OpenEnv's core architectural principles: + +### Server-Side Task Definition and Execution +**Invariant**: Custom tasks must be defined and executed server-side only, within the Docker container environment boundary. + +- **Task classes** are instantiated inside the environment server (e.g., `server/custom/custom_tasks.py`) +- **Reward computation** happens exclusively server-side, following RFC 002's "Environment-Computed Rewards" principle +- **Task injection** occurs via Docker volume mounts or build-time inclusion, never via client-side code injection + +Example volume mount approach: +```bash +docker run -v /local/tasks:/opt/openenv/custom_tasks \ + -e OPENENV_TASK_MODULE=custom_tasks.my_task \ + browsergym-env:latest +``` + +The environment server loads the task class from the mounted volume, but all execution stays within the container. + +### Action Execution Model +**Invariant**: Custom tasks inherit the parent environment's action execution model and security boundaries. + +For BrowserGym environments: +- Custom tasks use the same Playwright-based action execution as official benchmarks +- JavaScript execution via `page.evaluate()` is inherited from BrowserGym's standard behavior +- This is **not** a violation of the "Dual API boundary" - agents receive actions as strings and have no direct access to `reset()`, `step()`, or `state()` methods +- Browser-level sandboxing (Same-Origin Policy, CSP) is BrowserGym's responsibility + +For other environment types: +- Coding environments: Actions execute in sandboxed shell/interpreter +- Game environments: Actions map to discrete game moves +- Chat environments: Actions are text messages with no simulation control + +### Dual API Boundary Compliance +**Critical**: Custom tasks must not expose simulation control to agents. + +Prohibited: +- ❌ Exposing `reset()`, `step()`, or `state()` via MCP tools or action strings +- ❌ Allowing agents to modify reward computation logic at runtime +- ❌ Providing agents with direct access to the environment's HTTP API + +Allowed: +- ✅ Domain-specific actions (browser clicks, shell commands, game moves) +- ✅ Observation data that reflects task state +- ✅ Standard BrowserGym/Playwright actions that manipulate the task environment + +## Implementation Plan +1. Define the `Task` schema/interface in `core`. +2. Update `Environment.reset` signature (backward compatible). +3. Implement a basic `TaskRegistry` with server-side task loading. +4. Document volume mount and build-time task injection patterns. +5. Refactor `BrowserGymEnvironment` to support the new `reset(task=...)` pattern, leveraging the custom task system logic. +6. Extend to other environments (Coding, TextArena) incrementally. + +## Security Considerations + +### Task Isolation +- Custom tasks run in the same Docker container as the environment server +- Task code has the same privileges as the environment (by design) +- Users mounting custom tasks should trust the task code (similar to mounting any code into a container) + +### Reward Integrity +- Reward functions are part of the task class definition (server-side) +- Agents cannot modify reward logic via actions +- Reward computation uses only server-side state (page DOM, file system, etc.) + +### Future Enhancements +- Sandboxed task validation before loading +- Task signature verification for shared task repositories +- Rate limiting for resource-intensive custom tasks diff --git a/tests/envs/demo_custom_env.py b/tests/envs/demo_custom_env.py new file mode 100644 index 000000000..33212c1e8 --- /dev/null +++ b/tests/envs/demo_custom_env.py @@ -0,0 +1,71 @@ +"""Demo script to see the custom BrowserGym environment in action. + +This runs with a visible browser window so you can see the task and actions. + +Run this: + source .venv/bin/activate + python3 tests/envs/demo_custom_env.py +""" + +import sys +import os +import time + +# Add project root to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from envs.browsergym_env.server.browsergym_environment import BrowserGymEnvironment +from browsergym_env.models import BrowserGymAction + +print("=" * 70) +print("Custom BrowserGym Environment - Visual Demo") +print("=" * 70) +print("\nOpening browser window with custom task...\n") + +# Create environment with visible browser +env = BrowserGymEnvironment( + benchmark="custom", + task_name="copy-paste", + headless=False, + viewport_width=1280, + viewport_height=720, +) + +# Reset and show initial state +print("Browser opened, loading task...") +obs = env.reset() +print(f"\nGoal: {obs.goal}") +print("Initial page loaded.\n") +time.sleep(3) # Give you time to see the page + +# Execute some actions step-by-step +print("Now executing actions (watch the browser):\n") + +actions = [ + ("Click on source text field", "click('#source-text')"), + ("Select all text (Ctrl+A)", "press('Control+A')"), + ("Copy text (Ctrl+C)", "press('Control+C')"), + ("Click on target field", "click('#target-text')"), + ("Paste text (Ctrl+V)", "press('Control+V')"), + ("Click submit button", "click('#submit-btn')"), +] + +for i, (description, action_str) in enumerate(actions, 1): + print(f"Step {i}: {description}") + action = BrowserGymAction(action_str=action_str) + obs = env.step(action) + print(f" Reward: {obs.reward}, Done: {obs.done}") + + time.sleep(2) + + if obs.done: + print("\nTask completed.") + print(f"Total reward: {env.state.cum_reward}") + break + +print("\nKeeping browser open for 5 seconds...") +time.sleep(5) + +# Cleanup +env.close() +print("\nBrowser closed. Demo complete.") diff --git a/tests/envs/test_browsergym_custom.py b/tests/envs/test_browsergym_custom.py new file mode 100644 index 000000000..f2b949a4c --- /dev/null +++ b/tests/envs/test_browsergym_custom.py @@ -0,0 +1,383 @@ +"""Unit tests for BrowserGym custom task system. + +This comprehensive pytest suite tests the HTTP client/server integration. +Requires the BrowserGym server to be running. + +Run with pytest (from project root): + source .venv/bin/activate + pytest tests/envs/test_browsergym_custom.py -v + +Note: This suite starts its own server via fixtures. The maintainers will +run this as part of the CI/CD pipeline. +""" + +import os +import sys +import subprocess +import time +import requests +import pytest + +from envs.browsergym_env.client import BrowserGymEnv +from envs.browsergym_env.models import BrowserGymAction + + +@pytest.fixture(scope="module") +def custom_server(): + """Starts the BrowserGym environment server with custom task support.""" + ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + PORT = 8012 + localhost = f"http://localhost:{PORT}" + + print(f"\n--- Starting BrowserGym custom task server on port {PORT} ---") + + server_env = { + **os.environ, + "BROWSERGYM_BENCHMARK": "custom", + "BROWSERGYM_TASK_NAME": "copy-paste", + "BROWSERGYM_HEADLESS": "true", + } + + gunicorn_command = [ + "gunicorn", + "-w", + "1", + "-k", + "uvicorn.workers.UvicornWorker", + "-b", + f"0.0.0.0:{PORT}", + "envs.browsergym_env.server.app:app", + ] + + server_process = subprocess.Popen( + gunicorn_command, + env=server_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + # Wait for server to become healthy + print("\n--- Waiting for custom task server to become healthy... ---") + is_healthy = False + for i in range(12): + try: + response = requests.get(f"{localhost}/health", timeout=5) + if response.status_code == 200: + is_healthy = True + print("✅ Custom task server is running and healthy!") + break + except requests.exceptions.RequestException: + print(f"Attempt {i + 1}/12: Server not ready, waiting 10 seconds...") + time.sleep(10) + + if not is_healthy: + print("❌ Server did not become healthy in time. Aborting.") + print("\n--- Server Logs ---") + stdout, stderr = server_process.communicate(timeout=5) + print("STDOUT:", stdout) + print("STDERR:", stderr) + try: + server_process.kill() + except ProcessLookupError: + pass + pytest.skip("Custom task server failed to start") + + yield localhost + + # Cleanup + print("\n--- Cleaning up custom task server ---") + try: + server_process.kill() + print("✅ Server process killed") + except ProcessLookupError: + print("✅ Server process was already killed") + + +class TestCustomTaskRegistration: + """Test custom task registration system.""" + + def test_task_registry_has_builtin_tasks(self, custom_server): + """Test that built-in custom tasks are registered.""" + # Import the registry to check task registration + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + from envs.browsergym_env.server.custom.custom_tasks import _CUSTOM_TASKS + + assert "copy-paste" in _CUSTOM_TASKS + assert "copy-paste-multitab" in _CUSTOM_TASKS + + def test_get_custom_task_class(self, custom_server): + """Test retrieving a custom task class.""" + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + from envs.browsergym_env.server.custom.custom_tasks import get_custom_task + + task_class = get_custom_task("copy-paste") + assert task_class is not None + assert hasattr(task_class, "_calculate_reward") + assert hasattr(task_class, "_check_done") + + def test_invalid_task_name_raises_error(self, custom_server): + """Test that requesting an invalid task raises ValueError.""" + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + from envs.browsergym_env.server.custom.custom_tasks import get_custom_task + + with pytest.raises(ValueError, match="Unknown custom task"): + get_custom_task("nonexistent-task") + + +class TestCustomTaskEnvironment: + """Test custom task environment functionality.""" + + def test_health_endpoint(self, custom_server): + """Test that the health endpoint works.""" + response = requests.get(f"{custom_server}/health") + assert response.status_code == 200 + assert "status" in response.json() + + def test_custom_task_reset(self, custom_server): + """Test that reset() works with custom tasks.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + result = env.reset() + + assert result.observation is not None + assert hasattr(result.observation, "text") + assert result.observation.goal is not None + assert "copy" in result.observation.goal.lower() + + def test_custom_task_step(self, custom_server): + """Test that step() works with custom tasks.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Execute a simple action + action = BrowserGymAction(action_str="click('#source')") + result = env.step(action) + + assert result.observation is not None + assert hasattr(result.observation, "reward") + assert hasattr(result.observation, "done") + + def test_custom_task_state(self, custom_server): + """Test that state() returns valid custom task state.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + state = env.state() + assert state.benchmark == "custom" + assert state.task_name in ["copy-paste", "copy-paste-multitab"] + assert state.step_count >= 0 + + +class TestActionParsing: + """Test action parsing logic in custom tasks.""" + + def test_click_action(self, custom_server): + """Test that click actions are parsed correctly.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Click action should work + action = BrowserGymAction(action_str="click('#source')") + result = env.step(action) + assert result.observation is not None + + def test_fill_action(self, custom_server): + """Test that fill actions are parsed correctly.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Fill action should work + action = BrowserGymAction(action_str="fill('#source', 'test text')") + result = env.step(action) + assert result.observation is not None + + def test_goto_action(self, custom_server): + """Test that goto actions are parsed correctly.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Goto action should work (though may fail for custom tasks without navigation) + action = BrowserGymAction(action_str="goto('about:blank')") + result = env.step(action) + assert result.observation is not None + + def test_press_action(self, custom_server): + """Test that keyboard press actions work.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Press action should work + action = BrowserGymAction(action_str="press('Enter')") + result = env.step(action) + assert result.observation is not None + + def test_scroll_action(self, custom_server): + """Test that scroll actions work.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Scroll down + action = BrowserGymAction(action_str="scroll('down')") + result = env.step(action) + assert result.observation is not None + + def test_javascript_fallback(self, custom_server): + """Test that unrecognized actions fall back to JavaScript execution.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Raw JavaScript should execute + action = BrowserGymAction(action_str="console.log('test')") + result = env.step(action) + assert result.observation is not None + + def test_malformed_action_handling(self, custom_server): + """Test that malformed actions are handled gracefully.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Malformed action should not crash the server + action = BrowserGymAction(action_str="click(invalid syntax") + result = env.step(action) + assert result.observation is not None + # Error should be reflected in observation metadata or error field + + +class TestObservationConversion: + """Test observation conversion from custom task to BrowserGym format.""" + + def test_observation_has_required_fields(self, custom_server): + """Test that observations contain all required fields.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + result = env.reset() + + obs = result.observation + assert hasattr(obs, "text") + assert hasattr(obs, "goal") + assert hasattr(obs, "done") + assert hasattr(obs, "reward") + assert hasattr(obs, "metadata") + + def test_observation_text_extraction(self, custom_server): + """Test that observation text is extracted from page.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + result = env.reset() + + # Should have some text content + assert result.observation.text is not None + assert len(result.observation.text) > 0 + + def test_reward_calculation(self, custom_server): + """Test that rewards are calculated correctly.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Take a step and check reward + action = BrowserGymAction(action_str="click('#source')") + result = env.step(action) + + assert result.observation.reward is not None + assert isinstance(result.observation.reward, (int, float)) + + def test_done_flag_detection(self, custom_server): + """Test that done flag is set appropriately.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + result = env.reset() + + # Initially should not be done + assert not result.observation.done + + # After max steps or success, should be done + # (This depends on task implementation) + + +class TestErrorHandling: + """Test error handling in custom task system.""" + + def test_invalid_task_name_at_startup(self): + """Test that invalid task name is handled at server startup.""" + # This would need to start a server with an invalid task name + # and verify it either fails gracefully or returns an error + pass + + def test_action_error_handling(self, custom_server): + """Test that action errors are captured in observations.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Try to click a non-existent element + action = BrowserGymAction(action_str="click('#nonexistent-element-xyz')") + result = env.step(action) + + # Should not crash, but may have error in metadata + assert result.observation is not None + + def test_max_steps_enforcement(self, custom_server): + """Test that max_steps limit is enforced.""" + env = BrowserGymEnv(base_url=custom_server, request_timeout_s=60) + env.reset() + + # Take many steps to hit the limit + for i in range(15): + action = BrowserGymAction(action_str="scroll('down')") + result = env.step(action) + + # Should eventually be done due to max_steps + if result.observation.done: + break + + # After many steps, should be done + assert env.state().step_count > 0 + + +class TestTaskImplementations: + """Test specific custom task implementations.""" + + def test_copy_paste_task_goal(self, custom_server): + """Test that copy-paste task has correct goal.""" + # Set up environment with copy-paste task + ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + PORT = 8013 + localhost = f"http://localhost:{PORT}" + + server_env = { + **os.environ, + "BROWSERGYM_BENCHMARK": "custom", + "BROWSERGYM_TASK_NAME": "copy-paste", + "BROWSERGYM_HEADLESS": "true", + } + + gunicorn_command = [ + "gunicorn", + "-w", + "1", + "-k", + "uvicorn.workers.UvicornWorker", + "-b", + f"0.0.0.0:{PORT}", + "envs.browsergym_env.server.app:app", + ] + + server_process = subprocess.Popen( + gunicorn_command, + env=server_env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + try: + # Wait for server + time.sleep(10) + + env = BrowserGymEnv(base_url=localhost, request_timeout_s=60) + result = env.reset() + + assert "copy" in result.observation.goal.lower() + assert "paste" in result.observation.goal.lower() + + finally: + try: + server_process.kill() + except ProcessLookupError: + pass diff --git a/tests/envs/test_custom_integration.py b/tests/envs/test_custom_integration.py new file mode 100644 index 000000000..bfc03b514 --- /dev/null +++ b/tests/envs/test_custom_integration.py @@ -0,0 +1,250 @@ +"""Integration test for custom BrowserGym environment. + +This test verifies the end-to-end functionality: +1. Environment can be created with custom tasks +2. HTML content loads correctly +3. Actions can be executed +4. Observations are returned properly +5. Rewards are calculated + +Run this test: + source .venv/bin/activate + python3 tests/envs/test_custom_integration.py +""" + +import sys +import os + +# Add src to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "src")) + + +def test_custom_env_creation(): + """Test that custom environment can be created.""" + print("Testing custom environment creation...") + try: + from envs.browsergym_env.server.browsergym_environment import ( + BrowserGymEnvironment, + ) + + env = BrowserGymEnvironment( + benchmark="custom", + task_name="copy-paste", + headless=True, + viewport_width=1280, + viewport_height=720, + ) + + print(" PASS: Custom environment created successfully") + return env + + except Exception as e: + print(f" FAIL: Failed to create environment: {e}") + import traceback + + traceback.print_exc() + return None + + +def test_env_reset(env): + """Test that environment can be reset.""" + print("\nTesting environment reset...") + try: + obs = env.reset() + + # Verify observation structure + assert hasattr(obs, "goal"), "Observation missing 'goal' field" + assert hasattr(obs, "text"), "Observation missing 'text' field" + assert hasattr(obs, "done"), "Observation missing 'done' field" + assert hasattr(obs, "reward"), "Observation missing 'reward' field" + + print(" PASS: Environment reset successful") + print(f" Goal: {obs.goal[:60]}...") + print(f" Page text length: {len(obs.text)} characters") + print(f" Initial done: {obs.done}") + + return True + + except Exception as e: + print(f" FAIL: Reset failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_env_step(env): + """Test that actions can be executed.""" + print("\nTesting environment step...") + try: + from envs.browsergym_env.models import BrowserGymAction + + # Try a simple action (click on source text) + action = BrowserGymAction(action_str="click('#source-text')") + obs = env.step(action) + + # Verify observation + assert hasattr(obs, "reward"), "Observation missing 'reward' field" + assert hasattr(obs, "done"), "Observation missing 'done' field" + + print(" PASS: Step executed successfully") + print(f" Reward: {obs.reward}") + print(f" Done: {obs.done}") + + return True + + except Exception as e: + print(f" FAIL: Step failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_env_state(env): + """Test that environment state is accessible.""" + print("\nTesting environment state...") + try: + state = env.state + + # Verify state structure + assert hasattr(state, "benchmark"), "State missing 'benchmark' field" + assert hasattr(state, "task_name"), "State missing 'task_name' field" + assert hasattr(state, "step_count"), "State missing 'step_count' field" + + assert state.benchmark == "custom", f"Expected benchmark='custom', got '{state.benchmark}'" + assert ( + state.task_name == "copy-paste" + ), f"Expected task_name='copy-paste', got '{state.task_name}'" + + print(" PASS: State accessible") + print(f" Benchmark: {state.benchmark}") + print(f" Task: {state.task_name}") + print(f" Step count: {state.step_count}") + + return True + + except Exception as e: + print(f" FAIL: State access failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_task_html_loading(): + """Test that task HTML is loaded correctly.""" + print("\nTesting HTML task loading...") + try: + from envs.browsergym_env.server.custom.custom_tasks import get_custom_task + + task = get_custom_task("copy-paste") + + # Verify task has URL + url = task._get_task_url() + assert url, "Task URL is empty" + print(f" PASS: Task URL: {url[:80]}...") + + # Verify goal description + goal = task._get_goal_description() + assert goal, "Goal description is empty" + assert len(goal) > 0, "Goal description is too short" + print(f" PASS: Goal: {goal[:80]}...") + + return True + + except Exception as e: + print(f" FAIL: HTML loading test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_multitab_task(): + """Test that multi-tab task is available.""" + print("\nTesting multi-tab task...") + try: + from envs.browsergym_env.server.custom.custom_tasks import get_custom_task + + task = get_custom_task("copy-paste-multitab") + url = task._get_task_url() + goal = task._get_goal_description() + + assert url, "Multi-tab task URL is empty" + assert goal, "Multi-tab task goal is empty" + + print(" PASS: Multi-tab task available") + print(f" Goal: {goal[:80]}...") + + return True + + except Exception as e: + print(f" FAIL: Multi-tab task test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def main(): + """Run all integration tests.""" + print("=" * 70) + print("Custom BrowserGym Environment - Integration Tests") + print("=" * 70) + print("\nThese tests verify the custom environment works end-to-end.") + print("This may take a minute as it starts a headless browser...\n") + + # Test 1: HTML task loading (no browser needed) + test_results = [] + test_results.append(("HTML Task Loading", test_task_html_loading())) + test_results.append(("Multi-tab Task", test_multitab_task())) + + # Test 2-5: Full environment tests (requires browser) + env = test_custom_env_creation() + if env: + test_results.insert(0, ("Environment Creation", True)) + test_results.append(("Environment Reset", test_env_reset(env))) + test_results.append(("Environment Step", test_env_step(env))) + test_results.append(("Environment State", test_env_state(env))) + + # Cleanup + try: + env.close() + print("\nEnvironment closed successfully") + except Exception as e: + print(f"\nEnvironment cleanup warning: {e}") + else: + test_results.insert(0, ("Environment Creation", False)) + print("\nSkipping browser tests due to environment creation failure") + + # Summary + print("\n" + "=" * 70) + print("Summary") + print("=" * 70) + + all_passed = True + for test_name, passed in test_results: + status = "✅ PASS" if passed else "❌ FAIL" + print(f"{status}: {test_name}") + if not passed: + all_passed = False + + print("\n" + "=" * 70) + if all_passed: + print("ALL INTEGRATION TESTS PASSED") + print("\nCustom BrowserGym environment is fully functional:") + print(" - HTML tasks load correctly") + print(" - Environment follows OpenEnv interface") + print(" - Actions execute properly") + print(" - Observations are returned correctly") + print(" - State is accessible") + else: + print("Some integration tests failed.") + print("Please review the errors above.") + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/envs/test_custom_validation.py b/tests/envs/test_custom_validation.py new file mode 100644 index 000000000..7f824af70 --- /dev/null +++ b/tests/envs/test_custom_validation.py @@ -0,0 +1,166 @@ +"""Simple validation test to verify custom task system works. + +Run this directly to check your contribution: + source .venv/bin/activate + python3 tests/envs/test_custom_validation.py +""" + +import sys +import os + +# Add project root to path for direct imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + + +def test_imports(): + """Verify all custom task modules can be imported.""" + try: + from envs.browsergym_env.server.custom.custom_base import ( # noqa: F401 + CustomBrowserGymEnvironment, + ) + from envs.browsergym_env.server.custom.custom_models import ( # noqa: F401 + CustomGymAction, + CustomGymObservation, + CustomGymState, + ) + from envs.browsergym_env.server.custom.custom_tasks import ( # noqa: F401 + _CUSTOM_TASKS, + get_custom_task, + register_custom_task, + ) + return True + except ImportError as e: + print(f"Import failed: {e}") + return False + + +def test_task_registration(): + """Verify custom tasks are registered.""" + try: + from envs.browsergym_env.server.custom.custom_tasks import ( + _CUSTOM_TASKS, + get_custom_task, + ) + + expected_tasks = ["copy-paste", "copy-paste-multitab"] + for task_name in expected_tasks: + if task_name not in _CUSTOM_TASKS: + print(f"Task '{task_name}' not registered") + return False + + get_custom_task("copy-paste") + return True + + except Exception as e: + print(f"Registration test failed: {e}") + return False + + +def test_task_interface(): + """Verify task classes implement required methods.""" + try: + from envs.browsergym_env.server.custom.custom_tasks import get_custom_task + + task_instance = get_custom_task("copy-paste") + required_methods = [ + "_get_task_url", + "_get_goal_description", + "_calculate_reward", + "_check_done", + "reset", + "step", + "close", + ] + + for method_name in required_methods: + if not hasattr(task_instance, method_name): + print(f" Missing method: {method_name}") + return False + + return True + + except Exception as e: + print(f" Interface test failed: {e}") + return False + + +def test_models(): + """Verify data models can be instantiated.""" + print("\nTesting model instantiation...") + try: + from envs.browsergym_env.server.custom.custom_models import ( + CustomGymAction, + CustomGymObservation, + CustomGymState, + ) + + # Test Action + action = CustomGymAction(action_str="click('#button')") + print(f" PASS: CustomGymAction: {action.action_str}") + + # Test State + state = CustomGymState( + episode_id="test-123", + step_count=5, + benchmark="custom", + task_name="copy-paste", + max_steps=10, + ) + print(f" PASS: CustomGymState: episode={state.episode_id}, step={state.step_count}") + + # Test Observation + obs = CustomGymObservation( + text="Sample page", + goal="Copy text from source to target", + done=False, + reward=0.5, + ) + print(f" PASS: CustomGymObservation: reward={obs.reward}") + + return True + + except Exception as e: + print(f" FAIL: Model test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def main(): + """Run all validation tests.""" + print("Custom BrowserGym Task System - Validation Tests\n") + + tests = [ + ("Imports", test_imports), + ("Task Registration", test_task_registration), + ("Task Interface", test_task_interface), + ("Model Instantiation", test_models), + ] + + results = [] + for test_name, test_func in tests: + try: + results.append((test_name, test_func())) + except Exception as e: + print(f"{test_name} crashed: {e}") + results.append((test_name, False)) + + print("\nResults:") + all_passed = True + for test_name, passed in results: + status = "✅" if passed else "❌" + print(f"{status} {test_name}") + if not passed: + all_passed = False + + if all_passed: + print("\n✅ All tests passed") + else: + print("\n❌ Some tests failed") + + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main())