From 4b9f2e61c7eaaa55dbfccf3029ec31b2242478a9 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Thu, 4 Dec 2025 11:14:16 -0800 Subject: [PATCH] feat(mcp): allow passing --console-level --- docs/src/api/class-consolemessage.md | 2 +- packages/playwright-client/types/types.d.ts | 2 +- packages/playwright-core/types/types.d.ts | 2 +- packages/playwright/src/mcp/browser/config.ts | 26 +- packages/playwright/src/mcp/browser/tab.ts | 47 ++- .../src/mcp/browser/tools/console.ts | 4 +- packages/playwright/src/mcp/config.d.ts | 7 + packages/playwright/src/mcp/program.ts | 5 +- .../playwright/src/mcp/test/browserBackend.ts | 2 +- ...cp-browser-eval-fill-the-form-repeat0.json | 182 ++++++----- ...cp-browser-eval-fill-the-form-repeat1.json | 164 +++++----- ...cp-browser-eval-fill-the-form-repeat2.json | 303 +++++++----------- tests/mcp/console.spec.ts | 62 +++- 13 files changed, 448 insertions(+), 360 deletions(-) diff --git a/docs/src/api/class-consolemessage.md b/docs/src/api/class-consolemessage.md index b712bca267cdb..044d070342598 100644 --- a/docs/src/api/class-consolemessage.md +++ b/docs/src/api/class-consolemessage.md @@ -140,7 +140,7 @@ The text of the console message. ## method: ConsoleMessage.type * since: v1.8 * langs: js, python -- returns: <[ConsoleMessageType]<"log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd">> +- returns: <[ConsoleMessageType]<"log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd">> ## method: ConsoleMessage.type * since: v1.8 diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index d6ff2f0aceead..861d7a0f66f3b 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -18861,7 +18861,7 @@ export interface ConsoleMessage { */ text(): string; - type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd"; + type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd"; /** * The web worker or service worker that produced this console message, if any. Note that console messages from web diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index d6ff2f0aceead..861d7a0f66f3b 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -18861,7 +18861,7 @@ export interface ConsoleMessage { */ text(): string; - type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"timeEnd"; + type(): "log"|"debug"|"info"|"error"|"warning"|"dir"|"dirxml"|"table"|"trace"|"clear"|"startGroup"|"startGroupCollapsed"|"endGroup"|"assert"|"profile"|"profileEnd"|"count"|"time"|"timeEnd"; /** * The web worker or service worker that produced this console message, if any. Note that console messages from web diff --git a/packages/playwright/src/mcp/browser/config.ts b/packages/playwright/src/mcp/browser/config.ts index dc83efa9f0a6b..1099f3d281e0f 100644 --- a/packages/playwright/src/mcp/browser/config.ts +++ b/packages/playwright/src/mcp/browser/config.ts @@ -39,6 +39,7 @@ export type CLIOptions = { cdpEndpoint?: string; cdpHeader?: Record; config?: string; + consoleLevel?: 'error' | 'warning' | 'info' | 'debug'; device?: string; executablePath?: string; grantPermissions?: string[]; @@ -81,6 +82,9 @@ export const defaultConfig: FullConfig = { viewport: null, }, }, + console: { + level: 'info', + }, network: { allowedOrigins: undefined, blockedOrigins: undefined, @@ -104,6 +108,9 @@ export type FullConfig = Config & { launchOptions: NonNullable; contextOptions: NonNullable; }, + console: { + level: 'error' | 'warning' | 'info' | 'debug'; + }, network: NonNullable, saveTrace: boolean; server: NonNullable, @@ -241,6 +248,9 @@ export function configFromCLIOptions(cliOptions: CLIOptions): Config { allowedHosts: cliOptions.allowedHosts, }, capabilities: cliOptions.caps as ToolCapability[], + console: { + level: cliOptions.consoleLevel, + }, network: { allowedOrigins: cliOptions.allowedOrigins, blockedOrigins: cliOptions.blockedOrigins, @@ -274,6 +284,8 @@ function configFromEnv(): Config { options.cdpEndpoint = envToString(process.env.PLAYWRIGHT_MCP_CDP_ENDPOINT); options.cdpHeader = headerParser(process.env.PLAYWRIGHT_MCP_CDP_HEADERS, {}); options.config = envToString(process.env.PLAYWRIGHT_MCP_CONFIG); + if (process.env.PLAYWRIGHT_MCP_CONSOLE_LEVEL) + options.consoleLevel = enumParser<'error' | 'warning' | 'info' | 'debug'>('--console-level', ['error', 'warning', 'info', 'debug'], process.env.PLAYWRIGHT_MCP_CONSOLE_LEVEL); options.device = envToString(process.env.PLAYWRIGHT_MCP_DEVICE); options.executablePath = envToString(process.env.PLAYWRIGHT_MCP_EXECUTABLE_PATH); options.grantPermissions = commaSeparatedList(process.env.PLAYWRIGHT_MCP_GRANT_PERMISSIONS); @@ -287,8 +299,8 @@ function configFromEnv(): Config { if (initScript) options.initScript = [initScript]; options.isolated = envToBoolean(process.env.PLAYWRIGHT_MCP_ISOLATED); - if (process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES === 'omit') - options.imageResponses = 'omit'; + if (process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES) + options.imageResponses = enumParser<'allow' | 'omit'>('--image-responses', ['allow', 'omit'], process.env.PLAYWRIGHT_MCP_IMAGE_RESPONSES); options.sandbox = envToBoolean(process.env.PLAYWRIGHT_MCP_SANDBOX); options.outputDir = envToString(process.env.PLAYWRIGHT_MCP_OUTPUT_DIR); options.port = numberParser(process.env.PLAYWRIGHT_MCP_PORT); @@ -385,6 +397,10 @@ function mergeConfig(base: FullConfig, overrides: Config): FullConfig { ...pickDefined(base), ...pickDefined(overrides), browser, + console: { + ...pickDefined(base.console), + ...pickDefined(overrides.console), + }, network: { ...pickDefined(base.network), ...pickDefined(overrides.network), @@ -458,6 +474,12 @@ export function headerParser(arg: string | undefined, previous?: Record(name: string, options: T[], value: string): T { + if (!options.includes(value as T)) + throw new Error(`Invalid ${name}: ${value}. Valid values are: ${options.join(', ')}`); + return value as T; +} + function envToBoolean(value: string | undefined): boolean | undefined { if (value === 'true' || value === '1') return true; diff --git a/packages/playwright/src/mcp/browser/tab.ts b/packages/playwright/src/mcp/browser/tab.ts index d254b115bff31..9563bf7d29841 100644 --- a/packages/playwright/src/mcp/browser/tab.ts +++ b/packages/playwright/src/mcp/browser/tab.ts @@ -217,9 +217,9 @@ export class Tab extends EventEmitter { await this.waitForLoadState('load', { timeout: 5000 }); } - async consoleMessages(type?: 'error'): Promise { + async consoleMessages(level: ConsoleMessageLevel): Promise { await this._initializedPromise; - return this._consoleMessages.filter(message => type ? message.type === type : true); + return this._consoleMessages.filter(message => shouldIncludeMessage(level, message.type)); } async requests(): Promise> { @@ -244,7 +244,7 @@ export class Tab extends EventEmitter { }); if (tabSnapshot) { // Assign console message late so that we did not lose any to modal state. - tabSnapshot.consoleMessages = this._recentConsoleMessages; + tabSnapshot.consoleMessages = this._recentConsoleMessages.filter(message => shouldIncludeMessage(this.context.config.console.level, message.type)); this._recentConsoleMessages = []; } // If we failed to capture a snapshot this time, make sure we do a full one next time, @@ -317,7 +317,7 @@ export class Tab extends EventEmitter { } export type ConsoleMessage = { - type: ReturnType | undefined; + type: ReturnType; text: string; toString(): string; }; @@ -354,4 +354,43 @@ export function renderModalStates(modalStates: ModalState[]): string[] { return result; } +type ConsoleMessageType = ReturnType; +type ConsoleMessageLevel = 'error' | 'warning' | 'info' | 'debug'; +const consoleMessageLevels: ConsoleMessageLevel[] = ['error', 'warning', 'info', 'debug']; + +function shouldIncludeMessage(thresholdLevel: ConsoleMessageLevel, type: ConsoleMessageType): boolean { + const messageLevel = consoleLevelForMessageType(type); + return consoleMessageLevels.indexOf(messageLevel) <= consoleMessageLevels.indexOf(thresholdLevel); +} + +function consoleLevelForMessageType(type: ConsoleMessageType): ConsoleMessageLevel { + switch (type) { + case 'assert': + case 'error': + return 'error'; + case 'warning': + return 'warning'; + case 'count': + case 'dir': + case 'dirxml': + case 'info': + case 'log': + case 'table': + case 'time': + case 'timeEnd': + return 'info'; + case 'clear': + case 'debug': + case 'endGroup': + case 'profile': + case 'profileEnd': + case 'startGroup': + case 'startGroupCollapsed': + case 'trace': + return 'debug'; + default: + return 'info'; + } +} + const tabSymbol = Symbol('tabSymbol'); diff --git a/packages/playwright/src/mcp/browser/tools/console.ts b/packages/playwright/src/mcp/browser/tools/console.ts index b9bda093ac3c1..6643da31771b8 100644 --- a/packages/playwright/src/mcp/browser/tools/console.ts +++ b/packages/playwright/src/mcp/browser/tools/console.ts @@ -24,12 +24,12 @@ const console = defineTabTool({ title: 'Get console messages', description: 'Returns all console messages', inputSchema: z.object({ - onlyErrors: z.boolean().optional().describe('Only return error messages'), + level: z.enum(['error', 'warning', 'info', 'debug']).default('info').describe('Level of the console messages to return. Each level includes the messages of more severe levels. Defaults to "info".'), }), type: 'readOnly', }, handle: async (tab, params, response) => { - const messages = await tab.consoleMessages(params.onlyErrors ? 'error' : undefined); + const messages = await tab.consoleMessages(params.level); messages.map(message => response.addResult(message.toString())); }, }); diff --git a/packages/playwright/src/mcp/config.d.ts b/packages/playwright/src/mcp/config.d.ts index 9a04795ccf6aa..1f10ecdf34f0a 100644 --- a/packages/playwright/src/mcp/config.d.ts +++ b/packages/playwright/src/mcp/config.d.ts @@ -142,6 +142,13 @@ export type Config = { */ outputDir?: string; + console?: { + /** + * The level of console messages to return. Each level includes the messages of more severe levels. Defaults to "info". + */ + level?: 'error' | 'warning' | 'info' | 'debug'; + }, + network?: { /** * List of origins to allow the browser to request. Default is to allow all. Origins matching both `allowedOrigins` and `blockedOrigins` will be blocked. diff --git a/packages/playwright/src/mcp/program.ts b/packages/playwright/src/mcp/program.ts index 4635b7e1b3366..1c0389dcfb7eb 100644 --- a/packages/playwright/src/mcp/program.ts +++ b/packages/playwright/src/mcp/program.ts @@ -22,7 +22,7 @@ import { colors, ProgramOption } from 'playwright-core/lib/utilsBundle'; import { registry } from 'playwright-core/lib/server'; import * as mcpServer from './sdk/server'; -import { commaSeparatedList, dotenvFileLoader, headerParser, numberParser, resolutionParser, resolveCLIConfig, semicolonSeparatedList } from './browser/config'; +import { commaSeparatedList, dotenvFileLoader, enumParser, headerParser, numberParser, resolutionParser, resolveCLIConfig, semicolonSeparatedList } from './browser/config'; import { setupExitWatchdog } from './browser/watchdog'; import { contextFactory } from './browser/browserContextFactory'; import { ProxyBackend } from './sdk/proxyBackend'; @@ -43,6 +43,7 @@ export function decorateCommand(command: Command, version: string) { .option('--cdp-endpoint ', 'CDP endpoint to connect to.') .option('--cdp-header ', 'CDP headers to send with the connect request, multiple can be specified.', headerParser) .option('--config ', 'path to the configuration file.') + .option('--console-level ', 'level of console messages to return: "error", "warning", "info", "debug". Each level includes the messages of more severe levels.', enumParser.bind(null, '--console-level', ['error', 'warning', 'info', 'debug'])) .option('--device ', 'device to emulate, for example: "iPhone 15"') .option('--executable-path ', 'path to the browser executable.') .option('--extension', 'Connect to a running browser instance (Edge/Chrome only). Requires the "Playwright MCP Bridge" browser extension to be installed.') @@ -53,7 +54,7 @@ export function decorateCommand(command: Command, version: string) { .option('--init-page ', 'path to TypeScript file to evaluate on Playwright page object') .option('--init-script ', 'path to JavaScript file to add as an initialization script. The script will be evaluated in every page before any of the page\'s scripts. Can be specified multiple times.') .option('--isolated', 'keep the browser profile in memory, do not save it to disk.') - .option('--image-responses ', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".') + .option('--image-responses ', 'whether to send image responses to the client. Can be "allow" or "omit", Defaults to "allow".', enumParser.bind(null, '--image-responses', ['allow', 'omit'])) .option('--no-sandbox', 'disable the sandbox for all process types that are normally sandboxed.') .option('--output-dir ', 'path to the directory for output files.') .option('--port ', 'port to listen on for SSE transport.') diff --git a/packages/playwright/src/mcp/test/browserBackend.ts b/packages/playwright/src/mcp/test/browserBackend.ts index 12e8927ecddb7..7bed39f6bbfa2 100644 --- a/packages/playwright/src/mcp/test/browserBackend.ts +++ b/packages/playwright/src/mcp/test/browserBackend.ts @@ -95,7 +95,7 @@ async function generatePausedMessage(testInfo: TestInfo, context: playwright.Bro ); // Only print console errors when pausing on error, not when everything works as expected. let console = testInfo.errors.length ? await Tab.collectConsoleMessages(page) : []; - console = console.filter(msg => !msg.type || msg.type === 'error'); + console = console.filter(msg => msg.type === 'error'); if (console.length) { lines.push('- Console Messages:'); for (const message of console) diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json index 16c1d734a6bef..69e8c7ef746e9 100644 --- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json +++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat0.json @@ -1,27 +1,27 @@ { - "4c37062e059030f9c15a515aba7dea63145b1691": { + "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll help you navigate to the website, fill out the form to order a blue table and a green desk, and report back the total price and delivery time." + "text": "I'll help you navigate to the form, fill it out with the specified details, and report back the pricing and delivery information." } ] }, "usage": { - "input": 4390, - "output": 127 + "input": 4429, + "output": 120 } }, - "b09d261846c63220d3abdcee433a5987bce04d5f": { + "4498c58367500860e392c029b7f90d23ca898b50": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Navigate to the form page to order items", - "copilotToolCallId": "toolu_vrtx_01Nrr85ePXvh2wqRS6S47Lej" + "text": "Navigate to the form page to start the ordering process", + "copilotToolCallId": "toolu_vrtx_0129qd3zaawZfjyTyiNQ8dbY" }, { "type": "tool_call", @@ -29,23 +29,45 @@ "arguments": { "url": "http://localhost:/evals/fill-form.html" }, - "id": "toolu_vrtx_01Nrr85ePXvh2wqRS6S47Lej" + "id": "toolu_vrtx_0129qd3zaawZfjyTyiNQ8dbY" + } + ] + }, + "usage": { + "input": 4486, + "output": 93 + } + }, + "1f4cb362bd2e2d63916d74077a8cd478a0f0b1a5": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Get current page state to identify form fields for the first item (blue table)", + "copilotToolCallId": "toolu_vrtx_01Sqo3h7zZGVZYkVDWBrmhfg" + }, + { + "type": "tool_call", + "name": "browser_snapshot", + "arguments": {}, + "id": "toolu_vrtx_01Sqo3h7zZGVZYkVDWBrmhfg" } ] }, "usage": { - "input": 4454, - "output": 91 + "input": 5200, + "output": 67 } }, - "83da5a4a79d93a494d7f0f76d4758bc493d411f5": { + "54801f22b272e5a430fd56cc07ab4b11d87c6514": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Table' as the product type for the first item", - "copilotToolCallId": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi" + "text": "Select Table as the product type for the first order", + "copilotToolCallId": "toolu_vrtx_01KZtfPrpLh9Pg7D8hzZwfw5" }, { "type": "tool_call", @@ -57,23 +79,23 @@ "Table" ] }, - "id": "toolu_vrtx_011Vy4WSELJYN4U2tv9NgCLi" + "id": "toolu_vrtx_01KZtfPrpLh9Pg7D8hzZwfw5" } ] }, "usage": { - "input": 5166, - "output": 123 + "input": 5858, + "output": 120 } }, - "0a13ca0491d6745366d7b71a32b864412e848d78": { + "8585f97b99c7165ca864db490da286f768b8b883": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Blue' as the color for the table", - "copilotToolCallId": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY" + "text": "Select Blue as the color for the table", + "copilotToolCallId": "toolu_vrtx_01XGcSSgUwBCcgqLu11QqxRQ" }, { "type": "tool_call", @@ -85,23 +107,23 @@ "Blue" ] }, - "id": "toolu_vrtx_01AyhQJov7bZQ7dY9CmcZMeY" + "id": "toolu_vrtx_01XGcSSgUwBCcgqLu11QqxRQ" } ] }, "usage": { - "input": 5436, - "output": 120 + "input": 6125, + "output": 117 } }, - "0e1345afd5cefb63d40c0d9750aa6b73dfe36f38": { + "d78aa88a096ad38aea37e1b0b73aefd529845ab8": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Fill in the customer details for the order", - "copilotToolCallId": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH" + "text": "Fill in customer information for the order", + "copilotToolCallId": "toolu_vrtx_01Vk16c9EpV35aGtSjn2J6Pk" }, { "type": "tool_call", @@ -124,7 +146,7 @@ "name": "Street Address", "type": "textbox", "ref": "e19", - "value": "123 Main St, Anytown" + "value": "123 Main St" }, { "name": "City", @@ -146,23 +168,23 @@ } ] }, - "id": "toolu_vrtx_011xPGae2W6mMbh3TXpbqjaH" + "id": "toolu_vrtx_01Vk16c9EpV35aGtSjn2J6Pk" } ] }, "usage": { - "input": 5705, - "output": 348 + "input": 6391, + "output": 343 } }, - "90b95b16881abda623068841009641e51f32e7ae": { + "1380ec99c1a8290d52dd9d4bb008bc5add1efb54": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Click the Order button to submit the first order (blue table)", - "copilotToolCallId": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh" + "text": "Submit the order for the blue table", + "copilotToolCallId": "toolu_vrtx_011CLYRgGn3zonGBT97Wxr4r" }, { "type": "tool_call", @@ -171,23 +193,23 @@ "element": "Order button", "ref": "e29" }, - "id": "toolu_vrtx_01FwpyQSshn5YP7JyJbptXEh" + "id": "toolu_vrtx_011CLYRgGn3zonGBT97Wxr4r" } ] }, "usage": { - "input": 6163, - "output": 100 + "input": 6840, + "output": 95 } }, - "7160f78eeba943a8bf0daac205c6b183839401ca": { + "6ae203915486aa37b263ad3ad8af6af4a0c9e9f6": { "result": { "role": "assistant", "content": [ { "type": "text", "text": "Accept the confirmation dialog for the blue table order", - "copilotToolCallId": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y" + "copilotToolCallId": "toolu_vrtx_01BVGW6UrHvCKQ7v5xNPv1xv" }, { "type": "tool_call", @@ -195,23 +217,23 @@ "arguments": { "accept": true }, - "id": "toolu_vrtx_01LW5t6fGbyzj4ARj6wATf6y" + "id": "toolu_vrtx_01BVGW6UrHvCKQ7v5xNPv1xv" } ] }, "usage": { - "input": 6338, + "input": 7010, "output": 80 } }, - "80f4d8e16d86fb67f34dae4668d2ac84c71ac3ee": { + "9ec58f784b989e7cbbbde0656d22e6be1c5075dd": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Click 'Place Another Order' button to order the second item (green desk)", - "copilotToolCallId": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ" + "text": "Click the button to place another order for the green desk", + "copilotToolCallId": "toolu_vrtx_016rQY2JhLzVu3TpokRVqdor" }, { "type": "tool_call", @@ -220,23 +242,23 @@ "element": "Place Another Order button", "ref": "e58" }, - "id": "toolu_vrtx_01Y6z1hSQqTdsRs6VhoAkNrJ" + "id": "toolu_vrtx_016rQY2JhLzVu3TpokRVqdor" } ] }, "usage": { - "input": 7236, - "output": 106 + "input": 7897, + "output": 101 } }, - "8a7a2b461872a308c3c471069553fbb110b233e2": { + "83a5695faf3e865ff1654007db08a088bc2b9412": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Desk' as the product type for the second item", - "copilotToolCallId": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp" + "text": "Select Desk as the product type for the second order", + "copilotToolCallId": "toolu_vrtx_01WnDXifmCJWtEMduSaUjHU8" }, { "type": "tool_call", @@ -248,23 +270,23 @@ "Desk" ] }, - "id": "toolu_vrtx_01XFkp8Sb3zfxfSX56VB5Ukp" + "id": "toolu_vrtx_01WnDXifmCJWtEMduSaUjHU8" } ] }, "usage": { - "input": 7962, - "output": 125 + "input": 8618, + "output": 122 } }, - "aa6c7116bcda3f6079c1d67f86e781f745b9511f": { + "117e2c52b10e5288047ef65ac10a59c9e2275674": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Green' as the color for the desk", - "copilotToolCallId": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU" + "text": "Select Green as the color for the desk", + "copilotToolCallId": "toolu_vrtx_01462K6LBMFTWaVWPmegByzb" }, { "type": "tool_call", @@ -276,23 +298,23 @@ "Green" ] }, - "id": "toolu_vrtx_01BBnzzEU2bFJdGS8XeWbKbU" + "id": "toolu_vrtx_01462K6LBMFTWaVWPmegByzb" } ] }, "usage": { - "input": 8235, - "output": 120 + "input": 8888, + "output": 117 } }, - "a30b487338609e8a5d8417c1f90419956bb4b727": { + "ad76fc7dda0725834872ca9b8f4395722197026a": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Fill in the customer details for the second order", - "copilotToolCallId": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X" + "text": "Fill in customer information for the second order", + "copilotToolCallId": "toolu_vrtx_01Bonk7ZwhuGS7yc67tAqJMd" }, { "type": "tool_call", @@ -315,7 +337,7 @@ "name": "Street Address", "ref": "e19", "type": "textbox", - "value": "123 Main St, Anytown" + "value": "123 Main St" }, { "name": "City", @@ -337,23 +359,23 @@ } ] }, - "id": "toolu_vrtx_01W1QKYyFZvMCWhC3DwYQv2X" + "id": "toolu_vrtx_01Bonk7ZwhuGS7yc67tAqJMd" } ] }, "usage": { - "input": 8504, - "output": 269 + "input": 9154, + "output": 264 } }, - "7269d27383720010c65e7e530cf66cff736cb587": { + "2dd639bc23dfba249c89bacb6f09a6219538a99d": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Click the Order button to submit the second order (green desk)", - "copilotToolCallId": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9" + "text": "Submit the order for the green desk", + "copilotToolCallId": "toolu_vrtx_01B3Hfa3jbKn1278dRq9izUB" }, { "type": "tool_call", @@ -362,23 +384,23 @@ "element": "Order button", "ref": "e29" }, - "id": "toolu_vrtx_019EvisGfKGAHWu4Ehk6orM9" + "id": "toolu_vrtx_01B3Hfa3jbKn1278dRq9izUB" } ] }, "usage": { - "input": 8963, - "output": 100 + "input": 9604, + "output": 95 } }, - "28e8e2a0869e245c707a879c41bf7811648233ff": { + "df594f27d4f66356f38861d693b8f14a66ef0af4": { "result": { "role": "assistant", "content": [ { "type": "text", "text": "Accept the confirmation dialog for the green desk order", - "copilotToolCallId": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4" + "copilotToolCallId": "toolu_vrtx_01HPLerVExZZZ15M7UeJsh3X" }, { "type": "tool_call", @@ -386,23 +408,23 @@ "arguments": { "accept": true }, - "id": "toolu_vrtx_01MawbHWCqo6WBwPh8YcEBm4" + "id": "toolu_vrtx_01HPLerVExZZZ15M7UeJsh3X" } ] }, "usage": { - "input": 9138, + "input": 9774, "output": 80 } }, - "cc4e5987be7c007db351125c487c40e5413aebf1": { + "a6b8bcc69f1ddb715921d1814326fea5e5a96f22": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Report the total price and maximum delivery time for both orders", - "copilotToolCallId": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z" + "text": "Report the total price and maximum delivery time for both orders - blue table ($353.98) and green desk ($299.98)", + "copilotToolCallId": "toolu_vrtx_019b25HJgtNdXAz3QjmCnHYw" }, { "type": "tool_call", @@ -411,13 +433,13 @@ "price": 653.96, "deliveryDays": 3 }, - "id": "toolu_vrtx_01DahshwKgzha389UJ8rq29Z" + "id": "toolu_vrtx_019b25HJgtNdXAz3QjmCnHYw" } ] }, "usage": { - "input": 10037, - "output": 102 + "input": 10662, + "output": 117 } } } \ No newline at end of file diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json index 68b9b1d7cf583..67864d8c09633 100644 --- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json +++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat1.json @@ -1,27 +1,27 @@ { - "4c37062e059030f9c15a515aba7dea63145b1691": { + "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll help you complete this task. Let me start by navigating to the page and then filling out the order form." + "text": "I'll help you navigate to the form, fill it out, and order the items. Let me start by navigating to the URL and taking a snapshot to see the page structure." } ] }, "usage": { - "input": 4390, - "output": 114 + "input": 4429, + "output": 131 } }, - "e1e0bd5a8606f1711cc30f461f0c274f47c58792": { + "eb602996f263d0f29d50e40769506f6f47394046": { "result": { "role": "assistant", "content": [ { "type": "text", "text": "Navigate to the form page", - "copilotToolCallId": "toolu_vrtx_01CyPTAyQwbp3tJQ8HRHcZpU" + "copilotToolCallId": "toolu_vrtx_016zERgz28Qc3mWdBLi5V2Xp" }, { "type": "tool_call", @@ -29,23 +29,23 @@ "arguments": { "url": "http://localhost:/evals/fill-form.html" }, - "id": "toolu_vrtx_01CyPTAyQwbp3tJQ8HRHcZpU" + "id": "toolu_vrtx_016zERgz28Qc3mWdBLi5V2Xp" } ] }, "usage": { - "input": 4446, + "input": 4497, "output": 88 } }, - "7570cf20826150ea1678925d3c56cdaf6cc0ce0d": { + "ae111fb9371c47d04762f81458d5557be0ca760c": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Table' as the product type for the first order", - "copilotToolCallId": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK" + "text": "Select Table as the first product", + "copilotToolCallId": "toolu_vrtx_017WjVw9bpcfDw5892VkuT76" }, { "type": "tool_call", @@ -57,23 +57,23 @@ "Table" ] }, - "id": "toolu_vrtx_01R9atbVxke4YwYTPtDeMwuK" + "id": "toolu_vrtx_017WjVw9bpcfDw5892VkuT76" } ] }, "usage": { - "input": 5155, - "output": 123 + "input": 5206, + "output": 116 } }, - "1e9bce6dd706be6d7272ab962111b4e7eb5730d7": { + "08407b7115ec34eea2a164355fe32b570e965d41": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Blue' as the color for the table", - "copilotToolCallId": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT" + "text": "Select Blue as the color for the table", + "copilotToolCallId": "toolu_vrtx_01S7UqYcgtqNzZS4zrHwWHpu" }, { "type": "tool_call", @@ -85,23 +85,23 @@ "Blue" ] }, - "id": "toolu_vrtx_012uA5vSCDJPndFMhFP9z6AT" + "id": "toolu_vrtx_01S7UqYcgtqNzZS4zrHwWHpu" } ] }, "usage": { - "input": 5425, - "output": 120 + "input": 5469, + "output": 117 } }, - "40f9be0c8022103c35d156209c8f35be0861e84e": { + "ebcd1168f56b45b7402e8af39ab3532456911205": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Fill in the customer details for the order", - "copilotToolCallId": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv" + "text": "Fill in all the required customer information fields", + "copilotToolCallId": "toolu_vrtx_011f5gY6a6DjTX4w5Ddzdeeh" }, { "type": "tool_call", @@ -136,7 +136,7 @@ "name": "State/Province", "type": "textbox", "ref": "e25", - "value": "XYZ" + "value": "XYZ state" }, { "name": "ZIP/Postal Code", @@ -146,23 +146,23 @@ } ] }, - "id": "toolu_vrtx_01WGqTc3Hcmd1FTasnBTmwMv" + "id": "toolu_vrtx_011f5gY6a6DjTX4w5Ddzdeeh" } ] }, "usage": { - "input": 5694, - "output": 344 + "input": 5735, + "output": 345 } }, - "3f143bb307a820583389dcdc308cc4507dd72618": { + "a56bfd7a019d224004464dbe721e3cc0038886d6": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Submit the order for the blue table", - "copilotToolCallId": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx" + "text": "Click the Order button to submit the first item (blue table)", + "copilotToolCallId": "toolu_vrtx_013SEMdk1Hq4L5Co41uVazFk" }, { "type": "tool_call", @@ -171,23 +171,23 @@ "element": "Order button", "ref": "e29" }, - "id": "toolu_vrtx_01SLsAkw2y3eNVrRTPtAnfYx" + "id": "toolu_vrtx_013SEMdk1Hq4L5Co41uVazFk" } ] }, "usage": { - "input": 6144, - "output": 95 + "input": 6187, + "output": 100 } }, - "6e78e923857b521b4255a872588f577dee2d6795": { + "a238d85438b602deb2f96c474f8da1ba4829aba8": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Confirm the order for the blue table", - "copilotToolCallId": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF" + "text": "Accept the confirmation dialog for the blue table order", + "copilotToolCallId": "toolu_vrtx_018RwPsNMBXzSZ57LVQqCxrQ" }, { "type": "tool_call", @@ -195,23 +195,23 @@ "arguments": { "accept": true }, - "id": "toolu_vrtx_01MESzajZqXLjdsQiKvVgUhF" + "id": "toolu_vrtx_018RwPsNMBXzSZ57LVQqCxrQ" } ] }, "usage": { - "input": 6314, - "output": 79 + "input": 6362, + "output": 80 } }, - "1b5ccd39ae43f194ba622a7bb4bdf639e69ecec0": { + "e0b33dfe71582534d7ce4b55c37d7c649f42584a": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Click the button to place the second order (green desk)", - "copilotToolCallId": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg" + "text": "Click to place another order for the green desk", + "copilotToolCallId": "toolu_vrtx_018NyzMBhTPq4LisBkd4sQ4H" }, { "type": "tool_call", @@ -220,23 +220,23 @@ "element": "Place Another Order button", "ref": "e58" }, - "id": "toolu_vrtx_01UmUQhxxbPHM7UUXudex9Zg" + "id": "toolu_vrtx_018NyzMBhTPq4LisBkd4sQ4H" } ] }, "usage": { - "input": 7200, - "output": 101 + "input": 7252, + "output": 99 } }, - "4b74f6fa7afaa89403a0d6a70838971b04447ce6": { + "cc3f1e930399f534ab8b20c7eb2835c64fba1734": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Desk' as the product type for the second order", - "copilotToolCallId": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE" + "text": "Select Desk as the second product", + "copilotToolCallId": "toolu_vrtx_01Ltk3N21eQvg1KbBhmPRpFp" }, { "type": "tool_call", @@ -248,23 +248,23 @@ "Desk" ] }, - "id": "toolu_vrtx_01FRxMd3DbZstMP7cdMg3qNE" + "id": "toolu_vrtx_01Ltk3N21eQvg1KbBhmPRpFp" } ] }, "usage": { - "input": 7921, - "output": 125 + "input": 7971, + "output": 118 } }, - "0f35bef49760b5c9340ab795b04c5f4ccfb693af": { + "ab6e810564774b9fe4f7c834a4929ec7b7083836": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select 'Green' as the color for the desk", - "copilotToolCallId": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF" + "text": "Select Green as the color for the desk", + "copilotToolCallId": "toolu_vrtx_0176CAudgGEug4v6LMeDiGvm" }, { "type": "tool_call", @@ -276,23 +276,23 @@ "Green" ] }, - "id": "toolu_vrtx_01Ja8AwUYnJxFKNQbwvdTiQF" + "id": "toolu_vrtx_0176CAudgGEug4v6LMeDiGvm" } ] }, "usage": { - "input": 8194, - "output": 120 + "input": 8237, + "output": 117 } }, - "86eb224ddafacf54ef2fc05cf0db3893ef76e41e": { + "b8132fab3f5a79ff4d0b8c7167b0fc8bd1599a5c": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Fill in the customer details for the second order", - "copilotToolCallId": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv" + "text": "Fill in all the required customer information fields for the second order", + "copilotToolCallId": "toolu_vrtx_01ESimVVXfk6MxSe1yHmaywy" }, { "type": "tool_call", @@ -327,7 +327,7 @@ "name": "State/Province", "ref": "e25", "type": "textbox", - "value": "XYZ" + "value": "XYZ state" }, { "name": "ZIP/Postal Code", @@ -337,23 +337,23 @@ } ] }, - "id": "toolu_vrtx_012mxpfp6HJGg7pcLMxFt5fv" + "id": "toolu_vrtx_01ESimVVXfk6MxSe1yHmaywy" } ] }, "usage": { - "input": 8463, - "output": 265 + "input": 8503, + "output": 269 } }, - "d3c67880cd8b86dcc1311717192dbccd75f6f970": { + "b089efdaa78963cf4111a62cdcce00f6b0e37da4": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Submit the order for the green desk", - "copilotToolCallId": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa" + "text": "Click the Order button to submit the second item (green desk)", + "copilotToolCallId": "toolu_vrtx_01CCUVfDLfRxZQnmRHhoSRMw" }, { "type": "tool_call", @@ -362,23 +362,23 @@ "element": "Order button", "ref": "e29" }, - "id": "toolu_vrtx_01E69zhhreSssUMqZWhKaaPa" + "id": "toolu_vrtx_01CCUVfDLfRxZQnmRHhoSRMw" } ] }, "usage": { - "input": 8914, - "output": 95 + "input": 8959, + "output": 100 } }, - "c9719d6d50a2845be611b84f4882c960a639850b": { + "4ddf2575fb5623b5c9ccc345d065b0d6dd071681": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Confirm the order for the green desk", - "copilotToolCallId": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS" + "text": "Accept the confirmation dialog for the green desk order", + "copilotToolCallId": "toolu_vrtx_01Wgbaa8QXuE76APMMV3f9nE" }, { "type": "tool_call", @@ -386,23 +386,23 @@ "arguments": { "accept": true }, - "id": "toolu_vrtx_01XTAjsdAD5dNfef84Dq8pSS" + "id": "toolu_vrtx_01Wgbaa8QXuE76APMMV3f9nE" } ] }, "usage": { - "input": 9084, - "output": 79 + "input": 9134, + "output": 80 } }, - "beae0a9e066cec2c0e6c14b1bab32ea9a85c0448": { + "a6c801faaf41a1b323f5d01ac5b7a95c54a4f29c": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Report the total price and maximum delivery days for both orders. Blue table: $353.98, 3 days. Green desk: $299.98, 3 days. Total: $653.96, max delivery: 3 days.", - "copilotToolCallId": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk" + "text": "Report the combined results for both orders: blue table ($353.98) and green desk ($299.98), with a maximum delivery time of 3 days", + "copilotToolCallId": "toolu_vrtx_01Wn2WotjzPSUTWaYVu4Ymg8" }, { "type": "tool_call", @@ -411,13 +411,13 @@ "price": 653.96, "deliveryDays": 3 }, - "id": "toolu_vrtx_015FrLR8BJNkcxYXwgxkXkHk" + "id": "toolu_vrtx_01Wn2WotjzPSUTWaYVu4Ymg8" } ] }, "usage": { - "input": 9971, - "output": 143 + "input": 10025, + "output": 124 } } } \ No newline at end of file diff --git a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json index 2ffc3957804a8..5f95f099acf7c 100644 --- a/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json +++ b/tests/mcp/__cache__/copilot/mcp-browser-eval-fill-the-form-repeat2.json @@ -1,27 +1,27 @@ { - "4c37062e059030f9c15a515aba7dea63145b1691": { + "1f67d34de9dca04e9ec4d9d7b590b48b6e67fd08": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I'll help you complete this task. Let me start by navigating to the page and then filling out the form to order the items." + "text": "I'll help you navigate to the form, fill in the order details, and report back the price and delivery information." } ] }, "usage": { - "input": 4390, - "output": 117 + "input": 4429, + "output": 114 } }, - "fad5bf0fc534ecb1e403acfc02b0040f502dde65": { + "498aeb8d624d9373c7e5b5b1dff78828566dc900": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Navigate to the form page to start ordering items", - "copilotToolCallId": "toolu_vrtx_01LUFwA4XTZMZ1y1NHnRsMNx" + "text": "Navigate to the order form page", + "copilotToolCallId": "toolu_vrtx_01XPsGbYiofMbWmfgtdWBK7N" }, { "type": "tool_call", @@ -29,107 +29,41 @@ "arguments": { "url": "http://localhost:/evals/fill-form.html" }, - "id": "toolu_vrtx_01LUFwA4XTZMZ1y1NHnRsMNx" + "id": "toolu_vrtx_01XPsGbYiofMbWmfgtdWBK7N" } ] }, "usage": { - "input": 4449, - "output": 92 + "input": 4484, + "output": 89 } }, - "7fcc4660d74d7989887daa17f960a092cfa6d75d": { + "95fe071871583b1a5ddca2286d61680674dd9fd6": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Get current page state to see the form", - "copilotToolCallId": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA" - }, - { - "type": "tool_call", - "name": "browser_snapshot", - "arguments": {}, - "id": "toolu_vrtx_01S1TPuFGdY3PpCwQ1MTs7iA" - } - ] - }, - "usage": { - "input": 5162, - "output": 60 - } - }, - "f7864b65b2ac8c7b1371d54672bc5cea4ac0ee3a": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Table as the first product", - "copilotToolCallId": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Product Type dropdown", - "ref": "e7", - "values": [ - "Table" - ] - }, - "id": "toolu_vrtx_01U7qs1Y5D27RMjy5vigdaXR" - } - ] - }, - "usage": { - "input": 5813, - "output": 116 - } - }, - "7c78931db5b30156a7dc1e6612d870e8187a20d6": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Select Blue color for the table", - "copilotToolCallId": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi" - }, - { - "type": "tool_call", - "name": "browser_select_option", - "arguments": { - "element": "Color dropdown", - "ref": "e10", - "values": [ - "Blue" - ] - }, - "id": "toolu_vrtx_01LXzd6tHpmw9uCZGrNc3TSi" - } - ] - }, - "usage": { - "input": 6076, - "output": 115 - } - }, - "cd4bd5106418a526955afe6d3f7c7f2b53007346": { - "result": { - "role": "assistant", - "content": [ - { - "type": "text", - "text": "Fill in the customer information fields", - "copilotToolCallId": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV" + "text": "Fill in the order form for a blue table with customer details", + "copilotToolCallId": "toolu_vrtx_01E9xJ27rpZVDGuRiEKuYgFe" }, { "type": "tool_call", "name": "browser_fill_form", "arguments": { "fields": [ + { + "name": "Product Type", + "type": "combobox", + "ref": "e7", + "value": "Table" + }, + { + "name": "Color", + "type": "combobox", + "ref": "e10", + "value": "Blue" + }, { "name": "Full Name", "type": "textbox", @@ -146,19 +80,7 @@ "name": "Street Address", "type": "textbox", "ref": "e19", - "value": "123 Main St" - }, - { - "name": "City", - "type": "textbox", - "ref": "e22", - "value": "Anytown" - }, - { - "name": "State/Province", - "type": "textbox", - "ref": "e25", - "value": "XYZ state" + "value": "123 Main St, Anytown, XYZ state" }, { "name": "ZIP/Postal Code", @@ -168,159 +90,174 @@ } ] }, - "id": "toolu_vrtx_01YWjwfRNj3KaBujienmqBhV" + "id": "toolu_vrtx_01E9xJ27rpZVDGuRiEKuYgFe" } ] }, "usage": { - "input": 6340, - "output": 343 + "input": 5194, + "output": 354 } }, - "41b9db56c70fcd4f84f8d3de9de0c799dce8db88": { + "b33fa200303dc6e1c5e5d06e5566d11af9ee1100": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Submit the first order for blue table", - "copilotToolCallId": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB" + "text": "Check if City field needs to be filled before submitting", + "copilotToolCallId": "toolu_vrtx_01EyB4XTMFwAVCkJTVZegHBg" }, { "type": "tool_call", - "name": "browser_click", - "arguments": { - "element": "Order button", - "ref": "e29" - }, - "id": "toolu_vrtx_01D6FAeFSJeXh67RCcBh5JMB" + "name": "browser_snapshot", + "arguments": {}, + "id": "toolu_vrtx_01EyB4XTMFwAVCkJTVZegHBg" } ] }, "usage": { - "input": 6790, - "output": 95 + "input": 5642, + "output": 63 } }, - "e7da24b30926519b920dfff6d09d1f321ccb3ae7": { + "ddfd3471dba960dd013273a8ff6aa41c21a73ba0": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Confirm the order for blue table", - "copilotToolCallId": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD" + "text": "Fill in the missing City and State fields that are required", + "copilotToolCallId": "toolu_vrtx_01WQQgr577X5DKLNM66yaJvc" }, { "type": "tool_call", - "name": "browser_handle_dialog", + "name": "browser_fill_form", "arguments": { - "accept": true + "fields": [ + { + "name": "City", + "ref": "e22", + "type": "textbox", + "value": "Anytown" + }, + { + "name": "State/Province", + "ref": "e25", + "type": "textbox", + "value": "XYZ" + } + ] }, - "id": "toolu_vrtx_01TDuwkwXfeotUgRAu8fLUFD" + "id": "toolu_vrtx_01WQQgr577X5DKLNM66yaJvc" } ] }, "usage": { - "input": 6960, - "output": 78 + "input": 6345, + "output": 142 } }, - "b2200231c17c003417ced022a372a4b11edbf238": { + "6e1d42a5ece3458887bb317028560156de53d25d": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Click to place another order for the green desk", - "copilotToolCallId": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1" + "text": "Submit the order for the blue table", + "copilotToolCallId": "toolu_vrtx_01YQTMoAzSpRRw9J7kzMeAno" }, { "type": "tool_call", "name": "browser_click", "arguments": { - "element": "Place Another Order button", - "ref": "e58" + "element": "Order button", + "ref": "e29" }, - "id": "toolu_vrtx_014capLBfLNKY3rkD46m8bQ1" + "id": "toolu_vrtx_01YQTMoAzSpRRw9J7kzMeAno" } ] }, "usage": { - "input": 7848, - "output": 99 + "input": 6560, + "output": 95 } }, - "3eace161b63651422689b72dbf4e606a489163e6": { + "3090a7d97bc75644ff00d8b32d7fdbcf26c55f6a": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select Desk as the second product", - "copilotToolCallId": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi" + "text": "Confirm the order for the blue table", + "copilotToolCallId": "toolu_vrtx_01LvLQ372U35LRjMhjzg6Dme" }, { "type": "tool_call", - "name": "browser_select_option", + "name": "browser_handle_dialog", "arguments": { - "element": "Product Type dropdown", - "ref": "e7", - "values": [ - "Desk" - ] + "accept": true }, - "id": "toolu_vrtx_01CY7bczyF9uwULtuDbsv5Bi" + "id": "toolu_vrtx_01LvLQ372U35LRjMhjzg6Dme" } ] }, "usage": { - "input": 8567, - "output": 118 + "input": 6730, + "output": 79 } }, - "c59b631891d47364177fda353cfccb126dfbf570": { + "326f425b7b11f30fb2caef078ed93f7932923edc": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Select Green color for the desk", - "copilotToolCallId": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK" + "text": "Click on Place Another Order button to order the second item (green desk)", + "copilotToolCallId": "toolu_vrtx_01LEuTE7DgrAUMint4khguYr" }, { "type": "tool_call", - "name": "browser_select_option", + "name": "browser_click", "arguments": { - "element": "Color dropdown", - "ref": "e10", - "values": [ - "Green" - ] + "element": "Place Another Order button", + "ref": "e58" }, - "id": "toolu_vrtx_019fxgUwj5JiAJFrCzxGXPSK" + "id": "toolu_vrtx_01LEuTE7DgrAUMint4khguYr" } ] }, "usage": { - "input": 8833, - "output": 115 + "input": 7641, + "output": 104 } }, - "56e5cffa84ed757969f9abbbbe898dd6d0b2dbb6": { + "9128e6e3f63a0d1eaf2972336082bde3db8449ce": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Fill in the customer information for the second order", - "copilotToolCallId": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV" + "text": "Fill in the order form for a green desk with the same customer details", + "copilotToolCallId": "toolu_vrtx_01SboLoRCoYELQtRzgagA1EC" }, { "type": "tool_call", "name": "browser_fill_form", "arguments": { "fields": [ + { + "name": "Product Type", + "ref": "e7", + "type": "combobox", + "value": "Desk" + }, + { + "name": "Color", + "ref": "e10", + "type": "combobox", + "value": "Green" + }, { "name": "Full Name", "ref": "e13", @@ -337,7 +274,7 @@ "name": "Street Address", "ref": "e19", "type": "textbox", - "value": "123 Main St" + "value": "123 Main St, Anytown, XYZ state" }, { "name": "City", @@ -349,7 +286,7 @@ "name": "State/Province", "ref": "e25", "type": "textbox", - "value": "XYZ state" + "value": "XYZ" }, { "name": "ZIP/Postal Code", @@ -359,23 +296,23 @@ } ] }, - "id": "toolu_vrtx_016MMu8vjsBSdpwemMbNoXjV" + "id": "toolu_vrtx_01SboLoRCoYELQtRzgagA1EC" } ] }, "usage": { - "input": 9097, - "output": 266 + "input": 8365, + "output": 337 } }, - "d134b61bfaaea4e36415838a62db5cb13f888d20": { + "ccd12e74f5216b77104f1e2e6199519ef6bce9c0": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Submit the second order for green desk", - "copilotToolCallId": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf" + "text": "Submit the order for the green desk", + "copilotToolCallId": "toolu_vrtx_01HsUFw6GuXR6J4YYiNnEHMe" }, { "type": "tool_call", @@ -384,23 +321,23 @@ "element": "Order button", "ref": "e29" }, - "id": "toolu_vrtx_01TgMV6W9eKdLpPjDDDewJJf" + "id": "toolu_vrtx_01HsUFw6GuXR6J4YYiNnEHMe" } ] }, "usage": { - "input": 9550, + "input": 8931, "output": 95 } }, - "de09913d7981ad51cc18c9ad9ecdf7d1982bb858": { + "f3d9f4f90ffe9d6f68f0de098d16e50a7012bf6a": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Confirm the order for green desk", - "copilotToolCallId": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv" + "text": "Confirm the order for the green desk", + "copilotToolCallId": "toolu_vrtx_01EhMR6qGdFTfFJuC4GgTrzw" }, { "type": "tool_call", @@ -408,23 +345,23 @@ "arguments": { "accept": true }, - "id": "toolu_vrtx_018ksE83x6ekovoZuomxmrsv" + "id": "toolu_vrtx_01EhMR6qGdFTfFJuC4GgTrzw" } ] }, "usage": { - "input": 9720, - "output": 78 + "input": 9101, + "output": 79 } }, - "987bfeb4500a519fde51250611923095075f6dd9": { + "53eabaad4c0c8ed9499bada78403a56fa3ab6307": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "Report the total price for both items and maximum estimated delivery time", - "copilotToolCallId": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf" + "text": "Report the total price for both items and maximum estimated delivery. Blue table: $353.98, Green desk: $299.98. Total: $653.96. Both orders have 3 days estimated delivery.", + "copilotToolCallId": "toolu_vrtx_01VsdeN1JruLTtnoA2hNUu7u" }, { "type": "tool_call", @@ -433,13 +370,13 @@ "price": 653.96, "deliveryDays": 3 }, - "id": "toolu_vrtx_01JAZZ8bo7oMgM9FVCpBEvyf" + "id": "toolu_vrtx_01VsdeN1JruLTtnoA2hNUu7u" } ] }, "usage": { - "input": 10609, - "output": 103 + "input": 10013, + "output": 135 } } } \ No newline at end of file diff --git a/tests/mcp/console.spec.ts b/tests/mcp/console.spec.ts index 4deec26097a99..c627c08a5443b 100644 --- a/tests/mcp/console.spec.ts +++ b/tests/mcp/console.spec.ts @@ -99,6 +99,65 @@ test('recent console messages', async ({ client, server }) => { }); }); +test('recent console messages filter', async ({ startClient, server }) => { + server.setContent('/', ` + + + + + `, 'text/html'); + + const { client } = await startClient({ + args: ['--console-level', 'error'], + }); + + const response = parseResponse(await client.callTool({ + name: 'browser_navigate', + arguments: { + url: server.PREFIX, + }, + })); + + expect(response.consoleMessages).toContain('console.error'); + expect(response.consoleMessages).not.toContain('console.log'); +}); + +test('browser_console_messages default level', async ({ client, server }) => { + await client.callTool({ + name: 'browser_navigate', + arguments: { + url: server.HELLO_WORLD, + }, + }); + + await client.callTool({ + name: 'browser_evaluate', + arguments: { + function: `async () => { + console.debug("console.debug"); + console.log("console.log"); + console.warn("console.warn"); + console.error("console.error"); + setTimeout(() => { throw new Error("unhandled"); }, 0); + await fetch('/missing'); + }`, + }, + }); + + const response = parseResponse(await client.callTool({ + name: 'browser_console_messages', + })); + expect.soft(response.result).toContain('console.log'); + expect.soft(response.result).toContain('console.warn'); + expect.soft(response.result).toContain('console.error'); + expect.soft(response.result).toContain('Error: unhandled'); + expect.soft(response.result).toContain('404'); + expect.soft(response.result).not.toContain('console.debug'); +}); + test('browser_console_messages errors only', async ({ client, server }) => { await client.callTool({ name: 'browser_navigate', @@ -111,6 +170,7 @@ test('browser_console_messages errors only', async ({ client, server }) => { name: 'browser_evaluate', arguments: { function: `async () => { + console.debug("console.debug"); console.log("console.log"); console.warn("console.warn"); console.error("console.error"); @@ -123,7 +183,7 @@ test('browser_console_messages errors only', async ({ client, server }) => { const response = parseResponse(await client.callTool({ name: 'browser_console_messages', arguments: { - onlyErrors: true, + level: 'error', }, })); expect.soft(response.result).toContain('console.error');