fix(cli): display browser action results nicely instead of raw base64 data

.changeset/fix-browser-action-result-display.md

@@ -0,0 +1,6 @@
+---
+"@kilocode/cli": patch
+"kilo-code": patch
+---
+
+Fix browser action results displaying raw base64 screenshot data as hexadecimal garbage

cli/src/ui/messages/extension/say/SayBrowserActionResultMessage.tsx

@@ -1,14 +1,61 @@
 import React from "react"
 import { Box, Text } from "ink"
 import type { MessageComponentProps } from "../types.js"
-import { MarkdownText } from "../../../components/MarkdownText.js"
 import { useTheme } from "../../../../state/hooks/useTheme.js"
 
 /**
- * Display browser action results
+ * Parsed browser action result data
+ */
+interface BrowserActionResultData {
+	screenshot?: string
+	logs?: string
+	currentUrl?: string
+	currentMousePosition?: string
+	viewportWidth?: number
+	viewportHeight?: number
+}
+
+/**
+ * Parse browser action result from message text
+ */
+function parseBrowserActionResult(text: string | undefined): BrowserActionResultData | null {
+	if (!text) return null
+	try {
+		return JSON.parse(text) as BrowserActionResultData
+	} catch {
+		return null
+	}
+}
+
+/**
+ * Display browser action results in a readable format
+ * Parses the JSON data and shows meaningful info instead of raw base64 screenshot data
  */
 export const SayBrowserActionResultMessage: React.FC<MessageComponentProps> = ({ message }) => {
 	const theme = useTheme()
+	const result = parseBrowserActionResult(message.text)
+
+	// If we can't parse, show a simple message
+	if (!result) {
+		return (
+			<Box flexDirection="column" marginY={1}>
+				<Box>
+					<Text color={theme.semantic.info} bold>
+						🌐 Browser Action Result
+					</Text>
+				</Box>
+				<Box marginLeft={2} marginTop={1}>
+					<Text color={theme.ui.text.dimmed}>Browser action completed</Text>
+				</Box>
+			</Box>
+		)
+	}
+
+	const hasScreenshot = !!result.screenshot
+	const hasLogs = result.logs && result.logs.trim().length > 0
+	const hasUrl = !!result.currentUrl
+	const hasViewport = result.viewportWidth && result.viewportHeight
+
 	return (
 		<Box flexDirection="column" marginY={1}>
 			<Box>
@@ -17,11 +64,56 @@ export const SayBrowserActionResultMessage: React.FC<MessageComponentProps> = ({
 				</Text>
 			</Box>
 
-			{message.text && (
-				<Box marginLeft={2} marginTop={1}>
-					<MarkdownText>{message.text}</MarkdownText>
-				</Box>
-			)}
+			<Box flexDirection="column" marginLeft={2} marginTop={1}>
+				{/* Screenshot indicator */}
+				{hasScreenshot && (
+					<Box>
+						<Text color={theme.ui.text.dimmed}>📷 Screenshot captured</Text>
+					</Box>
+				)}
+
+				{/* Current URL */}
+				{hasUrl && (
+					<Box>
+						<Text color={theme.ui.text.dimmed}>
+							URL: <Text color={theme.markdown.link}>{result.currentUrl}</Text>
+						</Text>
+					</Box>
+				)}
+
+				{/* Viewport dimensions */}
+				{hasViewport && (
+					<Box>
+						<Text color={theme.ui.text.dimmed}>
+							Viewport: {result.viewportWidth}x{result.viewportHeight}
+						</Text>
+					</Box>
+				)}
+
+				{/* Cursor position */}
+				{result.currentMousePosition && (
+					<Box>
+						<Text color={theme.ui.text.dimmed}>Cursor: {result.currentMousePosition}</Text>
+					</Box>
+				)}
+
+				{/* Console logs */}
+				{hasLogs && (
+					<Box flexDirection="column" marginTop={1}>
+						<Text color={theme.ui.text.dimmed}>Console logs:</Text>
+						<Box marginLeft={2}>
+							<Text color={theme.ui.text.secondary}>{result.logs}</Text>
+						</Box>
+					</Box>
+				)}
+
+				{/* Fallback if no meaningful data */}
+				{!hasScreenshot && !hasLogs && !hasUrl && !hasViewport && (
+					<Box>
+						<Text color={theme.ui.text.dimmed}>Browser action completed</Text>
+					</Box>
+				)}
+			</Box>
 		</Box>
 	)
 }

cli/src/ui/messages/extension/say/__tests__/SayBrowserActionResultMessage.test.tsx

@@ -0,0 +1,198 @@
+import React from "react"
+import { render } from "ink-testing-library"
+import { describe, it, expect } from "vitest"
+import { SayBrowserActionResultMessage } from "../SayBrowserActionResultMessage.js"
+import type { ExtensionChatMessage } from "../../../../../types/messages.js"
+
+describe("SayBrowserActionResultMessage", () => {
+	const baseMessage: ExtensionChatMessage = {
+		ts: Date.now(),
+		type: "say",
+		say: "browser_action_result",
+	}
+
+	it("should display header for browser action result", () => {
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify({ screenshot: "data:image/png;base64,abc123" }),
+				}}
+			/>,
+		)
+		expect(lastFrame()).toContain("Browser Action Result")
+	})
+
+	it("should show screenshot indicator instead of base64 data", () => {
+		const browserResult = {
+			screenshot: "data:image/webp;base64,UklGRn44AABXRUJQVlA4...", // Simulated base64 data
+			logs: "",
+			currentUrl: "https://example.com",
+			viewportWidth: 1280,
+			viewportHeight: 800,
+		}
+
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify(browserResult),
+				}}
+			/>,
+		)
+
+		const output = lastFrame()
+
+		// Should show screenshot indicator
+		expect(output).toContain("Screenshot captured")
+
+		// Should NOT contain the base64 data
+		expect(output).not.toContain("UklGRn44AABXRUJQVlA4")
+		expect(output).not.toContain("data:image")
+
+		// Should show URL
+		expect(output).toContain("https://example.com")
+
+		// Should show viewport
+		expect(output).toContain("1280x800")
+	})
+
+	it("should display console logs when present", () => {
+		const browserResult = {
+			logs: "Console: Hello from the page\nError: Something went wrong",
+		}
+
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify(browserResult),
+				}}
+			/>,
+		)
+
+		const output = lastFrame()
+		expect(output).toContain("Console logs:")
+		expect(output).toContain("Hello from the page")
+		expect(output).toContain("Something went wrong")
+	})
+
+	it("should display cursor position when present", () => {
+		const browserResult = {
+			currentMousePosition: "500,300",
+		}
+
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify(browserResult),
+				}}
+			/>,
+		)
+
+		expect(lastFrame()).toContain("Cursor: 500,300")
+	})
+
+	it("should handle empty result gracefully", () => {
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify({}),
+				}}
+			/>,
+		)
+
+		expect(lastFrame()).toContain("Browser action completed")
+	})
+
+	it("should handle invalid JSON gracefully", () => {
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: "not valid json",
+				}}
+			/>,
+		)
+
+		expect(lastFrame()).toContain("Browser action completed")
+	})
+
+	it("should handle missing text gracefully", () => {
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: undefined,
+				}}
+			/>,
+		)
+
+		expect(lastFrame()).toContain("Browser action completed")
+	})
+
+	it("should not show logs section when logs are empty", () => {
+		const browserResult = {
+			screenshot: "data:image/png;base64,abc",
+			logs: "",
+		}
+
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify(browserResult),
+				}}
+			/>,
+		)
+
+		expect(lastFrame()).not.toContain("Console logs:")
+	})
+
+	it("should not show logs section when logs are only whitespace", () => {
+		const browserResult = {
+			screenshot: "data:image/png;base64,abc",
+			logs: "   \n\t  ",
+		}
+
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify(browserResult),
+				}}
+			/>,
+		)
+
+		expect(lastFrame()).not.toContain("Console logs:")
+	})
+
+	it("should display all available info together", () => {
+		const browserResult = {
+			screenshot: "data:image/png;base64,abc123",
+			logs: "Page loaded",
+			currentUrl: "https://test.com/page",
+			currentMousePosition: "100,200",
+			viewportWidth: 1920,
+			viewportHeight: 1080,
+		}
+
+		const { lastFrame } = render(
+			<SayBrowserActionResultMessage
+				message={{
+					...baseMessage,
+					text: JSON.stringify(browserResult),
+				}}
+			/>,
+		)
+
+		const output = lastFrame()
+		expect(output).toContain("Screenshot captured")
+		expect(output).toContain("https://test.com/page")
+		expect(output).toContain("1920x1080")
+		expect(output).toContain("100,200")
+		expect(output).toContain("Page loaded")
+	})
+})