Browser Use 2.0: backend emits executedCoordinate for click/hover; FE consumes it; remove fragile session-status parsing; ensure ResizeObserver cleanup; all tests passing (#8941)

Author: hannesrudolph

src/core/tools/browserActionTool.ts

@@ -183,17 +183,17 @@ export async function browserActionTool(
 
 				cline.consecutiveMistakeCount = 0
 
-				await cline.say(
-					"browser_action",
-					JSON.stringify({
-						action: action as BrowserAction,
-						coordinate,
-						text,
-						size,
-					} satisfies ClineSayBrowserAction),
-					undefined,
-					false,
-				)
+				// Prepare say payload; include executedCoordinate for pointer actions
+				const sayPayload: ClineSayBrowserAction & { executedCoordinate?: string } = {
+					action: action as BrowserAction,
+					coordinate,
+					text,
+					size,
+				}
+				if ((action === "click" || action === "hover") && processedCoordinate) {
+					sayPayload.executedCoordinate = processedCoordinate
+				}
+				await cline.say("browser_action", JSON.stringify(sayPayload), undefined, false)
 
 				switch (action) {
 					case "click":

src/shared/ExtensionMessage.ts

@@ -439,6 +439,7 @@ export interface ClineSayBrowserAction {
 	coordinate?: string
 	size?: string
 	text?: string
+	executedCoordinate?: string
 }
 
 export type BrowserActionResult = {

webview-ui/src/components/chat/BrowserActionRow.tsx

@@ -151,13 +151,13 @@ const BrowserActionRow = memo(({ message, nextMessage, actionIndex, totalActions
 			case "launch":
 				return `Launched browser`
 			case "click":
-				return `Clicked at: ${getViewportCoordinate(browserAction.coordinate)}`
+				return `Clicked at: ${browserAction.executedCoordinate || getViewportCoordinate(browserAction.coordinate)}`
 			case "type":
 				return `Typed: ${browserAction.text}`
 			case "press":
 				return `Pressed key: ${prettyKey(browserAction.text)}`
 			case "hover":
-				return `Hovered at: ${getViewportCoordinate(browserAction.coordinate)}`
+				return `Hovered at: ${browserAction.executedCoordinate || getViewportCoordinate(browserAction.coordinate)}`
 			case "scroll_down":
 				return "Scrolled down"
 			case "scroll_up":

webview-ui/src/components/chat/BrowserSessionRow.tsx

@@ -73,6 +73,7 @@ const prettyKey = (k?: string): string => {
 
 const getBrowserActionText = (
 	action: BrowserAction,
+	executedCoordinate?: string,
 	coordinate?: string,
 	text?: string,
 	size?: string,
@@ -113,7 +114,7 @@ const getBrowserActionText = (
 		case "launch":
 			return `Launched browser`
 		case "click":
-			return `Clicked at: ${getViewportCoordinate(coordinate)}`
+			return `Clicked at: ${executedCoordinate || getViewportCoordinate(coordinate)}`
 		case "type":
 			return `Typed: ${text}`
 		case "press":
@@ -123,7 +124,7 @@ const getBrowserActionText = (
 		case "scroll_up":
 			return "Scrolled up"
 		case "hover":
-			return `Hovered at: ${getViewportCoordinate(coordinate)}`
+			return `Hovered at: ${executedCoordinate || getViewportCoordinate(coordinate)}`
 		case "resize":
 			return `Resized to: ${size?.split(/[x,]/).join(" x ")}`
 		case "close":
@@ -623,6 +624,7 @@ const BrowserSessionRow = memo((props: BrowserSessionRowProps) => {
 										<span>
 											{getBrowserActionText(
 												action.action,
+												pages[currentPageIndex]?.mousePosition,
 												action.coordinate,
 												action.text,
 												pageSize,

webview-ui/src/components/chat/ChatView.tsx

@@ -972,10 +972,6 @@ const ChatViewComponent: React.ForwardRefRenderFunction<ChatViewRef, ChatViewPro
 			if (messages[i].ask === "browser_action_launch") {
 				return i
 			}
-			// Also check for browser_session_status as a fallback indicator
-			if (messages[i].say === "browser_session_status" && messages[i].text?.includes("opened")) {
-				return i
-			}
 		}
 		return -1
 	}, [messages])

webview-ui/src/components/chat/TaskHeader.tsx

@@ -103,23 +103,20 @@ const TaskHeader = ({
 		for (let i = 0; i < msgs.length; i++) {
 			const m = msgs[i] as any
 			if (m?.ask === "browser_action_launch") return i
-			if (m?.say === "browser_session_status" && typeof m.text === "string" && m.text.includes("opened")) {
-				return i
-			}
 		}
 		return -1
 	}, [clineMessages])
 
 	const showBrowserGlobe = browserSessionStartIndex !== -1 || !!isBrowserSessionActive
 
 	const condenseButton = (
-			<LucideIconButton
-				title={t("chat:task.condenseContext")}
-				icon={FoldVertical}
-				disabled={buttonsDisabled}
-				onClick={() => currentTaskItem && handleCondenseContext(currentTaskItem.id)}
-			/>
-		)
+		<LucideIconButton
+			title={t("chat:task.condenseContext")}
+			icon={FoldVertical}
+			disabled={buttonsDisabled}
+			onClick={() => currentTaskItem && handleCondenseContext(currentTaskItem.id)}
+		/>
+	)
 
 	const hasTodos = todos && Array.isArray(todos) && todos.length > 0
 
@@ -282,7 +279,7 @@ const TaskHeader = ({
 												{t("chat:task.contextWindow")}
 											</th>
 											<td className="font-light align-top">
-									<div className={`max-w-md -mt-1.5 flex flex-nowrap gap-1`}>
+												<div className={`max-w-md -mt-1.5 flex flex-nowrap gap-1`}>
 													<ContextWindowProgress
 														contextWindow={contextWindow}
 														contextTokens={contextTokens || 0}