diff --git a/.changeset/empty-spoons-float.md b/.changeset/empty-spoons-float.md new file mode 100644 index 000000000..51f4a82c5 --- /dev/null +++ b/.changeset/empty-spoons-float.md @@ -0,0 +1,5 @@ +--- +"@browserbasehq/stagehand": minor +--- + +Added a `stagehand.history` array which stores an array of `act`, `extract`, `observe`, and `goto` calls made. Since this history array is stored on the `StagehandPage` level, it will capture methods even if indirectly called by an agent. diff --git a/evals/evals.config.json b/evals/evals.config.json index dd4f4e770..e872253d8 100644 --- a/evals/evals.config.json +++ b/evals/evals.config.json @@ -1,5 +1,9 @@ { "tasks": [ + { + "name": "history", + "categories": ["combination"] + }, { "name": "expect_act_timeout", "categories": ["act"] diff --git a/evals/tasks/history.ts b/evals/tasks/history.ts new file mode 100644 index 000000000..6dbfcc1c5 --- /dev/null +++ b/evals/tasks/history.ts @@ -0,0 +1,54 @@ +import { initStagehand } from "@/evals/initStagehand"; +import { EvalFunction } from "@/types/evals"; + +export const history: EvalFunction = async ({ modelName, logger }) => { + const { stagehand, initResponse } = await initStagehand({ + modelName, + logger, + }); + + const { debugUrl, sessionUrl } = initResponse; + + await stagehand.page.goto("https://docs.stagehand.dev"); + + await stagehand.page.act("click on the 'Quickstart' tab"); + + await stagehand.page.extract("Extract the title of the page"); + + await stagehand.page.observe("Find all links on the page"); + + const history = stagehand.history; + + const hasCorrectNumberOfEntries = history.length === 4; + + const hasNavigateEntry = history[0].method === "navigate"; + const hasActEntry = history[1].method === "act"; + const hasExtractEntry = history[2].method === "extract"; + const hasObserveEntry = history[3].method === "observe"; + + const allEntriesHaveTimestamps = history.every( + (entry) => + typeof entry.timestamp === "string" && entry.timestamp.length > 0, + ); + const allEntriesHaveResults = history.every( + (entry) => entry.result !== undefined, + ); + + await stagehand.close(); + + const success = + hasCorrectNumberOfEntries && + hasNavigateEntry && + hasActEntry && + hasExtractEntry && + hasObserveEntry && + allEntriesHaveTimestamps && + allEntriesHaveResults; + + return { + _success: success, + debugUrl, + sessionUrl, + logs: logger.getLogs(), + }; +}; diff --git a/lib/StagehandPage.ts b/lib/StagehandPage.ts index 44b6c87aa..dc7daa781 100644 --- a/lib/StagehandPage.ts +++ b/lib/StagehandPage.ts @@ -6,6 +6,7 @@ import { Page, defaultExtractSchema } from "../types/page"; import { ExtractOptions, ExtractResult, + HistoryEntry, ObserveOptions, ObserveResult, } from "../types/stagehand"; @@ -39,6 +40,11 @@ export class StagehandPage { private userProvidedInstructions?: string; private waitForCaptchaSolves: boolean; private initialized: boolean = false; + private _history: Array = []; + + public get history(): ReadonlyArray { + return this._history; + } constructor( page: PlaywrightPage, @@ -294,6 +300,8 @@ export class StagehandPage { ? await this.api.goto(url, options) : await target.goto(url, options); + this.addToHistory("navigate", { url, options }, result); + if (this.waitForCaptchaSolves) { try { await this.waitForCaptchaSolve(1000); @@ -440,6 +448,19 @@ export class StagehandPage { } } + private addToHistory( + method: HistoryEntry["method"], + parameters: unknown, + result?: unknown, + ): void { + this._history.push({ + method, + parameters, + result: result ?? null, + timestamp: new Date().toISOString(), + }); + } + async act( actionOrOptions: string | ActOptions | ObserveResult, ): Promise { @@ -501,6 +522,7 @@ export class StagehandPage { if (this.api) { const result = await this.api.act(actionOrOptions); await this._refreshPageFromAPI(); + this.addToHistory("act", actionOrOptions, result); return result; } @@ -534,7 +556,7 @@ export class StagehandPage { }); // `useVision` is no longer passed to the handler - return this.actHandler + const result = await this.actHandler .act({ action, llmClient, @@ -569,6 +591,10 @@ export class StagehandPage { action: action, }; }); + + this.addToHistory("act", actionOrOptions, result); + + return result; } async extract( @@ -582,10 +608,14 @@ export class StagehandPage { // check if user called extract() with no arguments if (!instructionOrOptions) { + let result: ExtractResult; if (this.api) { - return this.api.extract({}); + result = await this.api.extract({}); + } else { + result = await this.extractHandler.extract(); } - return this.extractHandler.extract(); + this.addToHistory("extract", instructionOrOptions, result); + return result; } const options: ExtractOptions = @@ -615,7 +645,9 @@ export class StagehandPage { } if (this.api) { - return this.api.extract(options); + const result = await this.api.extract(options); + this.addToHistory("extract", instructionOrOptions, result); + return result; } const requestId = Math.random().toString(36).substring(2); @@ -643,7 +675,7 @@ export class StagehandPage { }, }); - return this.extractHandler + const result = await this.extractHandler .extract({ instruction, schema, @@ -676,6 +708,10 @@ export class StagehandPage { throw e; }); + + this.addToHistory("extract", instructionOrOptions, result); + + return result; } async observe( @@ -729,7 +765,9 @@ export class StagehandPage { } if (this.api) { - return this.api.observe(options); + const result = await this.api.observe(options); + this.addToHistory("observe", instructionOrOptions, result); + return result; } const requestId = Math.random().toString(36).substring(2); @@ -761,7 +799,7 @@ export class StagehandPage { }, }); - return this.observeHandler + const result = await this.observeHandler .observe({ instruction, llmClient, @@ -802,6 +840,10 @@ export class StagehandPage { throw e; }); + + this.addToHistory("observe", instructionOrOptions, result); + + return result; } async getCDPClient(): Promise { diff --git a/lib/index.ts b/lib/index.ts index b026c19b0..b2ec2de49 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -28,6 +28,7 @@ import { AgentConfig, StagehandMetrics, StagehandFunctionName, + HistoryEntry, } from "../types/stagehand"; import { StagehandContext } from "./StagehandContext"; import { StagehandPage } from "./StagehandPage"; @@ -899,6 +900,16 @@ export class Stagehand { }, }; } + + public get history(): ReadonlyArray { + if (!this.stagehandPage) { + throw new Error( + "History is only available after a page has been initialized", + ); + } + + return this.stagehandPage.history; + } } export * from "../types/browser"; diff --git a/types/stagehand.ts b/types/stagehand.ts index 8ae451706..db3499780 100644 --- a/types/stagehand.ts +++ b/types/stagehand.ts @@ -244,3 +244,10 @@ export enum StagehandFunctionName { EXTRACT = "EXTRACT", OBSERVE = "OBSERVE", } + +export interface HistoryEntry { + method: "act" | "extract" | "observe" | "navigate"; + parameters: unknown; + result: unknown; + timestamp: string; +}