diff --git a/.changeset/fix-browser-action-result-display.md b/.changeset/fix-browser-action-result-display.md new file mode 100644 index 00000000000..bf10a8498ca --- /dev/null +++ b/.changeset/fix-browser-action-result-display.md @@ -0,0 +1,6 @@ +--- +"@kilocode/cli": patch +"kilo-code": patch +--- + +Fix browser action results displaying raw base64 screenshot data as hexadecimal garbage diff --git a/cli/src/ui/messages/extension/say/SayBrowserActionResultMessage.tsx b/cli/src/ui/messages/extension/say/SayBrowserActionResultMessage.tsx index c27d3130231..6a8eef3d9dc 100644 --- a/cli/src/ui/messages/extension/say/SayBrowserActionResultMessage.tsx +++ b/cli/src/ui/messages/extension/say/SayBrowserActionResultMessage.tsx @@ -1,14 +1,61 @@ import React from "react" import { Box, Text } from "ink" import type { MessageComponentProps } from "../types.js" -import { MarkdownText } from "../../../components/MarkdownText.js" import { useTheme } from "../../../../state/hooks/useTheme.js" /** - * Display browser action results + * Parsed browser action result data + */ +interface BrowserActionResultData { + screenshot?: string + logs?: string + currentUrl?: string + currentMousePosition?: string + viewportWidth?: number + viewportHeight?: number +} + +/** + * Parse browser action result from message text + */ +function parseBrowserActionResult(text: string | undefined): BrowserActionResultData | null { + if (!text) return null + try { + return JSON.parse(text) as BrowserActionResultData + } catch { + return null + } +} + +/** + * Display browser action results in a readable format + * Parses the JSON data and shows meaningful info instead of raw base64 screenshot data */ export const SayBrowserActionResultMessage: React.FC = ({ message }) => { const theme = useTheme() + const result = parseBrowserActionResult(message.text) + + // If we can't parse, show a simple message + if (!result) { + return ( + + + + 🌐 Browser Action Result + + + + Browser action completed + + + ) + } + + const hasScreenshot = !!result.screenshot + const hasLogs = result.logs && result.logs.trim().length > 0 + const hasUrl = !!result.currentUrl + const hasViewport = result.viewportWidth && result.viewportHeight + return ( @@ -17,11 +64,56 @@ export const SayBrowserActionResultMessage: React.FC = ({ - {message.text && ( - - {message.text} - - )} + + {/* Screenshot indicator */} + {hasScreenshot && ( + + 📷 Screenshot captured + + )} + + {/* Current URL */} + {hasUrl && ( + + + URL: {result.currentUrl} + + + )} + + {/* Viewport dimensions */} + {hasViewport && ( + + + Viewport: {result.viewportWidth}x{result.viewportHeight} + + + )} + + {/* Cursor position */} + {result.currentMousePosition && ( + + Cursor: {result.currentMousePosition} + + )} + + {/* Console logs */} + {hasLogs && ( + + Console logs: + + {result.logs} + + + )} + + {/* Fallback if no meaningful data */} + {!hasScreenshot && !hasLogs && !hasUrl && !hasViewport && ( + + Browser action completed + + )} + ) } diff --git a/cli/src/ui/messages/extension/say/__tests__/SayBrowserActionResultMessage.test.tsx b/cli/src/ui/messages/extension/say/__tests__/SayBrowserActionResultMessage.test.tsx new file mode 100644 index 00000000000..9a45519a584 --- /dev/null +++ b/cli/src/ui/messages/extension/say/__tests__/SayBrowserActionResultMessage.test.tsx @@ -0,0 +1,198 @@ +import React from "react" +import { render } from "ink-testing-library" +import { describe, it, expect } from "vitest" +import { SayBrowserActionResultMessage } from "../SayBrowserActionResultMessage.js" +import type { ExtensionChatMessage } from "../../../../../types/messages.js" + +describe("SayBrowserActionResultMessage", () => { + const baseMessage: ExtensionChatMessage = { + ts: Date.now(), + type: "say", + say: "browser_action_result", + } + + it("should display header for browser action result", () => { + const { lastFrame } = render( + , + ) + expect(lastFrame()).toContain("Browser Action Result") + }) + + it("should show screenshot indicator instead of base64 data", () => { + const browserResult = { + screenshot: "data:image/webp;base64,UklGRn44AABXRUJQVlA4...", // Simulated base64 data + logs: "", + currentUrl: "https://example.com", + viewportWidth: 1280, + viewportHeight: 800, + } + + const { lastFrame } = render( + , + ) + + const output = lastFrame() + + // Should show screenshot indicator + expect(output).toContain("Screenshot captured") + + // Should NOT contain the base64 data + expect(output).not.toContain("UklGRn44AABXRUJQVlA4") + expect(output).not.toContain("data:image") + + // Should show URL + expect(output).toContain("https://example.com") + + // Should show viewport + expect(output).toContain("1280x800") + }) + + it("should display console logs when present", () => { + const browserResult = { + logs: "Console: Hello from the page\nError: Something went wrong", + } + + const { lastFrame } = render( + , + ) + + const output = lastFrame() + expect(output).toContain("Console logs:") + expect(output).toContain("Hello from the page") + expect(output).toContain("Something went wrong") + }) + + it("should display cursor position when present", () => { + const browserResult = { + currentMousePosition: "500,300", + } + + const { lastFrame } = render( + , + ) + + expect(lastFrame()).toContain("Cursor: 500,300") + }) + + it("should handle empty result gracefully", () => { + const { lastFrame } = render( + , + ) + + expect(lastFrame()).toContain("Browser action completed") + }) + + it("should handle invalid JSON gracefully", () => { + const { lastFrame } = render( + , + ) + + expect(lastFrame()).toContain("Browser action completed") + }) + + it("should handle missing text gracefully", () => { + const { lastFrame } = render( + , + ) + + expect(lastFrame()).toContain("Browser action completed") + }) + + it("should not show logs section when logs are empty", () => { + const browserResult = { + screenshot: "data:image/png;base64,abc", + logs: "", + } + + const { lastFrame } = render( + , + ) + + expect(lastFrame()).not.toContain("Console logs:") + }) + + it("should not show logs section when logs are only whitespace", () => { + const browserResult = { + screenshot: "data:image/png;base64,abc", + logs: " \n\t ", + } + + const { lastFrame } = render( + , + ) + + expect(lastFrame()).not.toContain("Console logs:") + }) + + it("should display all available info together", () => { + const browserResult = { + screenshot: "data:image/png;base64,abc123", + logs: "Page loaded", + currentUrl: "https://test.com/page", + currentMousePosition: "100,200", + viewportWidth: 1920, + viewportHeight: 1080, + } + + const { lastFrame } = render( + , + ) + + const output = lastFrame() + expect(output).toContain("Screenshot captured") + expect(output).toContain("https://test.com/page") + expect(output).toContain("1920x1080") + expect(output).toContain("100,200") + expect(output).toContain("Page loaded") + }) +})