From 73aff9f8663a7a76cb3b4240012b1bece9308e19 Mon Sep 17 00:00:00 2001 From: Celve Date: Thu, 5 Nov 2026 15:00:04 +0800 Subject: [PATCH] Run Harbor computer-1 on a desktop its image already runs, behind an env var computer-1 starts its own Xvfb, XFCE and Chromium and treats Chromium's exit as the session's end, so it can only drive a browser. HARBOR_CUA_DESKTOP=existing patches it, on a trial's first run, to use the image's display on :1: no desktop or browser of its own, liveness from the X socket, no browser reset or recovery, zoom and the browser actions refused and cut from the generic provider's prompt, a settle after each screen-changing action, PNG screenshots optionally resized in the sandbox, and a cap on the screenshots one request carries. It touches private Harbor names pinned at harbor-miles-v0.20.0, and is a stopgap until computer-1 takes desktop options. --- examples/experimental/harbor/README.md | 6 +- .../harbor/harbor_agent_function.py | 176 ++++++++++++++++- examples/experimental/harbor/launch_common.py | 4 + .../harbor/test_harbor_contract.py | 181 +++++++++++++++++- .../experimental/harbor/test_launch_common.py | 4 + 5 files changed, 368 insertions(+), 3 deletions(-) diff --git a/examples/experimental/harbor/README.md b/examples/experimental/harbor/README.md index 989fc1a9b28..55cfad2407f 100644 --- a/examples/experimental/harbor/README.md +++ b/examples/experimental/harbor/README.md @@ -113,7 +113,11 @@ the `--sglang-enable-token-space-processor`, `--shared-media-dir` and `--use-miles-router` that session images require. Qwen3-VL clicks on a 0-999 grid while computer-1's generic provider reads pixels, so set `HARBOR_CUA_COORDINATE_SPACE=normalized_0_999`, which patches that provider to prompt for and rescale a 0-999 grid. Other computer-1 settings, -such as the desktop size, go in `HARBOR_AGENT_KWARGS`. +such as the desktop size, go in `HARBOR_AGENT_KWARGS`. computer-1 starts its own Xvfb, XFCE +and Chromium; to drive a desktop the task image already runs on `:1` instead, set +`HARBOR_CUA_DESKTOP=existing`, with `HARBOR_CUA_SETTLE_SEC`, `HARBOR_CUA_SCREENSHOT_SIZE` and +`HARBOR_CUA_KEEP_SCREENSHOTS` as `harbor_agent_function.py`'s header describes. The image needs +ImageMagick's `import` and `convert` and `xdotool` on its PATH. ## Validation diff --git a/examples/experimental/harbor/harbor_agent_function.py b/examples/experimental/harbor/harbor_agent_function.py index dff59fce98e..20f4a1be0bf 100644 --- a/examples/experimental/harbor/harbor_agent_function.py +++ b/examples/experimental/harbor/harbor_agent_function.py @@ -63,6 +63,20 @@ ``normalized_0_999`` makes Harbor's generic computer provider prompt for, and rescale, clicks on a 0..999 grid; unset leaves Harbor as it is + HARBOR_CUA_DESKTOP ``existing`` makes computer-1 drive the desktop its + image already runs on ``:1``: no Xvfb, XFCE, VNC or + Chromium, liveness from the X socket, PNG screenshots, + and zoom and the browser actions refused; unset leaves + Harbor as it is. With it: + HARBOR_CUA_SETTLE_SEC + seconds to wait after each screen-changing action + (default 2.0) + HARBOR_CUA_SCREENSHOT_SIZE + ``WIDTHxHEIGHT`` the screenshot is resized to in the + sandbox (default: the desktop's own size) + HARBOR_CUA_KEEP_SCREENSHOTS + most screenshots one request carries, the current + one included (default: Harbor's own trimming) Failure semantics: a verdict is returned as-is; every episode that ends without one scores 0 with a named ``exit_status`` (``TimeLimitExceeded``, @@ -80,9 +94,11 @@ import math import os import re +import shlex from collections.abc import Callable from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath +from types import SimpleNamespace from typing import Any from miles.rollout.agentic.credentials import PROVIDER_CREDENTIALS, resolve_provider_api_key @@ -150,6 +166,163 @@ def parse_on_grid(self, llm_response): _coordinate_space_applied = True +_BROWSER_ACTIONS = frozenset({"zoom", "navigate", "go_back", "go_forward"}) +_BROWSER_PROMPT_LINES = ( + " zoom, navigate,", + '\n "url": ,', + '\n "zoom_region": <[x0, y0, x1, y1] in desktop pixels, optional, used by zoom>,', + '\n- For "zoom": provide "zoom_region" as [x0, y0, x1, y1] in desktop pixels.' + "\n The NEXT screenshot is cropped (no resize) to that region, then auto-resets." + "\n Use this to inspect a small UI area at native pixel density.", + '\n- For "navigate": provide the destination URL in "url".', +) +_desktop_applied = False + + +def _screenshot_size() -> tuple[int, int] | None: + raw = os.getenv("HARBOR_CUA_SCREENSHOT_SIZE", "").strip() + if not raw: + return None + match = re.fullmatch(r"([1-9]\d*)x([1-9]\d*)", raw) + if match is None: + raise ValueError(f"HARBOR_CUA_SCREENSHOT_SIZE={raw!r}; expected WIDTHxHEIGHT") + return int(match[1]), int(match[2]) + + +def _images_in(prompt: Any) -> int: + turns = [prompt] if isinstance(prompt, dict) else prompt if isinstance(prompt, list) else [] + return sum( + isinstance(part, dict) and part.get("type") == "image_url" + for turn in turns + if isinstance(turn, dict) and isinstance(turn.get("content"), list) + for part in turn["content"] + ) + + +def apply_existing_desktop() -> None: + """Patch Harbor's computer-1 to drive the desktop its image runs when ``HARBOR_CUA_DESKTOP=existing``.""" + global _desktop_applied + desktop = os.getenv("HARBOR_CUA_DESKTOP", "").strip() + if not desktop or _desktop_applied: + return + if desktop != "existing": + raise ValueError(f"HARBOR_CUA_DESKTOP={desktop!r}; the only supported value is existing") + settle_sec = float(os.getenv("HARBOR_CUA_SETTLE_SEC", "2.0")) + size = _screenshot_size() + keep = _env_int("HARBOR_CUA_KEEP_SCREENSHOTS") + if keep is not None and keep < 1: + raise ValueError("HARBOR_CUA_KEEP_SCREENSHOTS must be at least 1: the request needs the current screen") + _patch_desktop_prompt() + _patch_desktop_session(settle_sec=settle_sec, size=size) + if keep is not None: + _patch_screenshot_history(keep=keep) + _desktop_applied = True + + +def _patch_desktop_prompt() -> None: + from harbor.agents.computer_1.providers import generic + + template = (generic._TEMPLATES_DIR / "computer-1-json.txt").read_text() + for line in _BROWSER_PROMPT_LINES: + if template.count(line) != 1: + raise ValueError(f"computer-1's prompt template no longer holds {line!r} exactly once") + template = template.replace(line, "") + from_agent = generic.GenericJsonProvider.from_agent.__func__ + + def from_agent_on_desktop(cls, agent): + provider = from_agent(cls, agent) + provider._prompt_template = template + provider.screenshot_format = "png" + return provider + + generic.GenericJsonProvider.from_agent = classmethod(from_agent_on_desktop) + + +def _patch_desktop_session(*, settle_sec: float, size: tuple[int, int] | None) -> None: + from harbor.agents.computer_1 import runtime + + session = runtime.Computer1Session + execute = session.execute + unchanged_screen = frozenset({"wait", "sleep", *runtime.TERMINAL_ACTION_TYPES}) + + async def start(self) -> None: + if self._started: + return + dirs = " ".join(shlex.quote(d) for d in (runtime._RUNTIME_DIR, runtime._SCREENSHOT_DIR, str(self._agent_dir))) + await self._exec( + runtime._bash_inline(f"mkdir -p {dirs}"), timeout_sec=60, label="mkdir runtime dirs", retries=2 + ) + await self._wait_for_x11() + self._started = True + + async def is_session_alive(self) -> bool: + try: + result = await self._raw_exec("test -S /tmp/.X11-unix/X1 && echo ok || echo down", timeout_sec=5) + except Exception: + return False + return "ok" in (result.stdout or "") + + async def reset(self) -> None: + raise RuntimeError("a desktop session has no browser to relaunch") + + async def no_recovery(self, action_type: str, exc: Exception) -> None: + return None + + async def execute_and_settle(self, action) -> dict[str, Any]: + if action.type in _BROWSER_ACTIONS: + raise runtime.RuntimeRequestError(action.type, 400, f"{action.type!r} is not available on this desktop") + result = await execute(self, action) + if action.type not in unchanged_screen and settle_sec > 0: + await asyncio.sleep(settle_sec) + return result + + async def fetch_screenshot(self, env_path) -> str: + target = str(env_path) + if PurePosixPath(target).suffix.lower() != ".png": + raise ValueError(f"a desktop session writes PNG screenshots, not {target!r}") + png = shlex.quote(f"{runtime._SCREENSHOT_DIR}/latest.png") + resize = f" && convert {png} -resize {size[0]}x{size[1]}! {png}" if size else "" + script = ( + f"set -e; export DISPLAY={runtime._DEFAULT_DISPLAY}; " + f"mkdir -p {shlex.quote(runtime._SCREENSHOT_DIR)} {shlex.quote(str(PurePosixPath(target).parent))}; " + f"import -window root {png}{resize}; cp {png} {shlex.quote(target)}" + ) + await self._exec( + runtime._bash_inline(script), timeout_sec=max(30, self._request_timeout_sec), label="screenshot" + ) + return target + + session.start = start + session.is_session_alive = is_session_alive + session.reset = reset + session._recover_chromium_if_needed = no_recovery + session.execute = execute_and_settle + session.fetch_screenshot = fetch_screenshot + + +def _patch_screenshot_history(*, keep: int) -> None: + from harbor.agents.computer_1.computer_1 import Computer1 + + query = Computer1._query_litellm + fallback_history = Computer1._fallback_message_history + + def trim(agent, messages: list[Any], prompt_images: int) -> None: + holder = SimpleNamespace(messages=messages) + agent._compactor._trim_old_screenshots(holder, keep_last=max(0, keep - prompt_images)) + + async def query_trimmed(self, chat, prompt, *args, **kwargs): + trim(self, chat.messages, _images_in(prompt)) + return await query(self, chat, prompt, *args, **kwargs) + + def fallback_history_trimmed(self): + history = fallback_history(self) + trim(self, history, int(self._enable_images and self._latest_screenshot_path is not None)) + return history + + Computer1._query_litellm = query_trimmed + Computer1._fallback_message_history = fallback_history_trimmed + + def _env_json_object(var: str) -> dict[str, Any]: value = json.loads(os.getenv(var, "") or "{}") if not isinstance(value, dict): @@ -582,6 +755,7 @@ async def run( # rewards. _ensure_provider_key() apply_coordinate_space() + apply_existing_desktop() config = build_trial_config(metadata, session_url, request_kwargs) try: diff --git a/examples/experimental/harbor/launch_common.py b/examples/experimental/harbor/launch_common.py index 465f2ca42d0..4c60a4d29ae 100644 --- a/examples/experimental/harbor/launch_common.py +++ b/examples/experimental/harbor/launch_common.py @@ -124,6 +124,10 @@ def harbor_env_vars(args) -> dict[str, str]: "HARBOR_AGENT_ALLOWED_HOSTS", "HARBOR_AGENT_IMPORT_PATH", "HARBOR_CUA_COORDINATE_SPACE", + "HARBOR_CUA_DESKTOP", + "HARBOR_CUA_SETTLE_SEC", + "HARBOR_CUA_SCREENSHOT_SIZE", + "HARBOR_CUA_KEEP_SCREENSHOTS", "HARBOR_AGENT_KWARGS", "HARBOR_VERIFIER_IMPORT_PATH", "HARBOR_VERIFIER_KWARGS", diff --git a/tests/fast/examples/experimental/harbor/test_harbor_contract.py b/tests/fast/examples/experimental/harbor/test_harbor_contract.py index 6b5b7439999..d51ffc2c15f 100644 --- a/tests/fast/examples/experimental/harbor/test_harbor_contract.py +++ b/tests/fast/examples/experimental/harbor/test_harbor_contract.py @@ -12,18 +12,29 @@ import asyncio import base64 import json +import shlex import threading from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import PurePosixPath +from types import SimpleNamespace import pytest harbor = pytest.importorskip("harbor") import harbor_agent_function as haf # noqa: E402 +from harbor.agents.computer_1 import runtime # noqa: E402 from harbor.agents.computer_1.computer_1 import Computer1, Computer1Chat, EpisodeLoggingPaths # noqa: E402 +from harbor.agents.computer_1.providers import generic # noqa: E402 from harbor.agents.computer_1.providers.base import image_url_part # noqa: E402 from harbor.agents.computer_1.providers.generic import GenericJsonProvider # noqa: E402 -from harbor.agents.computer_1.runtime import DisplayGeometry, normalize_completion_action # noqa: E402 +from harbor.agents.computer_1.runtime import ( # noqa: E402 + Computer1Session, + ComputerAction, + DisplayGeometry, + RuntimeRequestError, + normalize_completion_action, +) from harbor.agents.factory import AgentFactory # noqa: E402 from harbor.llms.base import LLMResponse # noqa: E402 from harbor.models.agent.name import AgentName # noqa: E402 @@ -140,6 +151,174 @@ def test_an_unknown_coordinate_space_stops_the_run(monkeypatch): haf.apply_coordinate_space() +_SESSION_PATCHED = ("start", "is_session_alive", "reset", "_recover_chromium_if_needed", "execute", "fetch_screenshot") +_DESKTOP_PATCHED = [ + *[(Computer1Session, name) for name in _SESSION_PATCHED], + (GenericJsonProvider, "from_agent"), + (Computer1, "_query_litellm"), + (Computer1, "_fallback_message_history"), +] +_X_SOCKET_ALIVE = "test -S /tmp/.X11-unix/X1 && echo ok || echo down" + + +@pytest.fixture +def harbor_restored(monkeypatch): + for owner, name in _DESKTOP_PATCHED: + monkeypatch.setattr(owner, name, owner.__dict__[name]) + monkeypatch.setattr(haf, "_desktop_applied", False, raising=False) + + +@pytest.fixture +def desktop(harbor_restored, tasks_dir, tmp_path, monkeypatch): + monkeypatch.setenv("HARBOR_CUA_DESKTOP", "existing") + + def build(session_url="http://s/sessions/s1/v1", **env): + for var, value in env.items(): + monkeypatch.setenv(var, value) + haf.apply_existing_desktop() + return _computer_1(session_url, tmp_path, monkeypatch) + + return build + + +def _recording_environment(): + commands: list[str] = [] + + async def exec(command, timeout_sec=None, user=None, **kwargs): + commands.append(command) + return SimpleNamespace(stdout="ok", stderr="", return_code=0) + + return SimpleNamespace(exec=exec, default_user=None), commands + + +def _started(agent): + environment, commands = _recording_environment() + asyncio.run(agent.setup(environment)) + started = list(commands) + commands.clear() + return agent._session, started, commands + + +def _image_turn(text: str) -> dict: + return {"role": "user", "content": [{"type": "text", "text": text}, image_url_part("data:image/png;base64,AA==")]} + + +def _images(messages) -> int: + return sum( + part.get("type") == "image_url" for m in messages if isinstance(m["content"], list) for part in m["content"] + ) + + +def test_on_an_existing_desktop_the_session_starts_no_desktop_or_browser(desktop): + _, started, _ = _started(desktop()) + + assert not [c for c in started for word in ("Xvfb", "startxfce4", "x11vnc", "chromium") if word in c] + assert len(started) == 2 + assert "mkdir -p" in started[0] + assert started[1] == "test -S /tmp/.X11-unix/X1 && echo ok || echo wait" + + +def test_on_an_existing_desktop_liveness_reads_only_the_x_socket(desktop): + session, _, commands = _started(desktop()) + + assert asyncio.run(session.is_session_alive()) is True + assert commands == [_X_SOCKET_ALIVE] + assert asyncio.run(session._recover_chromium_if_needed("click", RuntimeError())) is None + with pytest.raises(RuntimeError, match="no browser"): + asyncio.run(session.reset()) + + +def test_on_an_existing_desktop_browser_actions_are_refused_and_screen_changes_settle(desktop, monkeypatch): + session, _, _ = _started(desktop(HARBOR_CUA_SETTLE_SEC="0.25")) + sleeps = [] + + async def recorded_sleep(delay, *args, **kwargs): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, "sleep", recorded_sleep) + + for action in (ComputerAction(type="navigate", url="https://example.com"), ComputerAction(type="zoom")): + with pytest.raises(RuntimeRequestError, match="not available"): + asyncio.run(session.execute(action)) + asyncio.run(session.execute(ComputerAction(type="click", x=5, y=5))) + asyncio.run(session.execute(ComputerAction(type="wait"))) + assert sleeps == [0.25, 1.0] + + +def test_on_an_existing_desktop_the_screenshot_is_a_resized_png(desktop): + agent = desktop(HARBOR_CUA_SCREENSHOT_SIZE="1280x720") + session, _, commands = _started(agent) + provider = GenericJsonProvider.from_agent(agent) + + asyncio.run(session.fetch_screenshot("/logs/agent/screenshot_1.png")) + png = "/tmp/computer_1-screenshots/latest.png" + assert ( + f"import -window root {png} && convert {png} -resize 1280x720! {png}; cp {png} " in shlex.split(commands[0])[2] + ) + with pytest.raises(ValueError, match="PNG"): + asyncio.run(session.fetch_screenshot("/logs/agent/screenshot_1.webp")) + assert provider.screenshot_format == "png" + assert '"navigate"' not in provider.record_text("task") and "zoom_region" not in provider.record_text("task") + + +def test_on_an_existing_desktop_a_request_carries_at_most_the_kept_screenshots(desktop, chat_server): + base_url, requests = chat_server + agent = desktop(f"{base_url}/sessions/s1/v1", HARBOR_CUA_KEEP_SCREENSHOTS="2") + agent._chat = chat = Computer1Chat(agent._llm) + for turn in range(3): + chat.messages.extend([_image_turn(f"screen {turn}"), {"role": "assistant", "content": "click"}]) + + asyncio.run(agent._query_litellm(chat, [_image_turn("now")], EpisodeLoggingPaths(None, None, None))) + agent._latest_screenshot_path = "/logs/agent/screenshot_4.png" + + assert _images(requests[0][2]["messages"]) == 2 + assert _images(agent._fallback_message_history()) == 1 + + +def test_without_a_desktop_harbor_is_untouched(harbor_restored, monkeypatch): + monkeypatch.delenv("HARBOR_CUA_DESKTOP", raising=False) + before = [owner.__dict__[name] for owner, name in _DESKTOP_PATCHED] + + haf.apply_existing_desktop() + + assert all(owner.__dict__[name] is kept for (owner, name), kept in zip(_DESKTOP_PATCHED, before, strict=True)) + + +@pytest.mark.parametrize( + "env, error", + [ + ({"HARBOR_CUA_DESKTOP": "xfce"}, "existing"), + ({"HARBOR_CUA_DESKTOP": "existing", "HARBOR_CUA_SCREENSHOT_SIZE": "1280"}, "WIDTHxHEIGHT"), + ({"HARBOR_CUA_DESKTOP": "existing", "HARBOR_CUA_KEEP_SCREENSHOTS": "0"}, "at least 1"), + ], +) +def test_a_bad_desktop_setting_stops_the_run(harbor_restored, monkeypatch, env, error): + for var, value in env.items(): + monkeypatch.setenv(var, value) + + with pytest.raises(ValueError, match=error): + haf.apply_existing_desktop() + + +def test_the_private_harbor_names_the_desktop_patch_touches_exist(tasks_dir, tmp_path, monkeypatch): + agent = _computer_1("http://s/sessions/s1/v1", tmp_path, monkeypatch) + session = Computer1Session(_recording_environment()[0], PurePosixPath("/logs/agent")) + + assert all( + hasattr(runtime, name) for name in ("_RUNTIME_DIR", "_SCREENSHOT_DIR", "_DEFAULT_DISPLAY", "_bash_inline") + ) + assert (generic._TEMPLATES_DIR / "computer-1-json.txt").is_file() + assert hasattr(GenericJsonProvider.from_agent(agent), "_prompt_template") + assert all( + callable(getattr(session, name)) + for name in ("_recover_chromium_if_needed", "_wait_for_x11", "_exec", "_raw_exec") + ) + assert all(hasattr(session, name) for name in ("_started", "_agent_dir", "_request_timeout_sec")) + assert all(callable(getattr(agent, name)) for name in ("_query_litellm", "_fallback_message_history")) + assert callable(agent._compactor._trim_old_screenshots) + assert all(hasattr(agent, name) for name in ("_enable_images", "_latest_screenshot_path")) + + def test_harbor_builds_computer_1_from_the_import_path_with_max_seq_len_set(tasks_dir, tmp_path, monkeypatch): agent = _computer_1("http://trainer:30000/sessions/s1/v1", tmp_path, monkeypatch) diff --git a/tests/fast/examples/experimental/harbor/test_launch_common.py b/tests/fast/examples/experimental/harbor/test_launch_common.py index 97370e85188..296b4c03d04 100644 --- a/tests/fast/examples/experimental/harbor/test_launch_common.py +++ b/tests/fast/examples/experimental/harbor/test_launch_common.py @@ -95,6 +95,10 @@ def test_agent_and_verifier_classes_are_forwarded_with_their_kwargs(monkeypatch) knobs = { "HARBOR_AGENT_IMPORT_PATH": "pkg.agent:MyComputer1", "HARBOR_CUA_COORDINATE_SPACE": "normalized_0_999", + "HARBOR_CUA_DESKTOP": "existing", + "HARBOR_CUA_SETTLE_SEC": "2.0", + "HARBOR_CUA_SCREENSHOT_SIZE": "1280x720", + "HARBOR_CUA_KEEP_SCREENSHOTS": "2", "HARBOR_AGENT_KWARGS": '{"coordinate_space": "normalized_0_999"}', "HARBOR_VERIFIER_IMPORT_PATH": "pkg.verify:HostVerifier", "HARBOR_VERIFIER_KWARGS": '{"read_limit_mb": 64}',