operator handler

.changeset/young-dots-fry.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": minor
+---
+
+Added native Stagehand agentic loop functionality. This allows you to build agentic workflows with a single prompt without using a computer-use model. To try it out, create a `stagehand.agent` without passing in a provider.

examples/operator-example.ts

@@ -0,0 +1,83 @@
+import { Stagehand } from "@/dist";
+import dotenv from "dotenv";
+import StagehandConfig from "@/stagehand.config";
+import chalk from "chalk";
+
+// Load environment variables
+dotenv.config();
+
+async function main() {
+  console.log(`\n${chalk.bold("Stagehand 🤘 Native Agent Example")}\n`);
+
+  // Initialize Stagehand
+  console.log(`${chalk.cyan("→")} Initializing Stagehand...`);
+  const stagehand = new Stagehand({
+    ...StagehandConfig,
+  });
+
+  await stagehand.init();
+  console.log(`${chalk.green("✓")} Stagehand initialized`);
+
+  try {
+    const page = stagehand.page;
+
+    console.log(`\n${chalk.magenta.bold("⚡ First Agent Execution")}`);
+
+    const agent = stagehand.agent({
+      instructions: `You are a helpful assistant that can use a web browser.
+      You are currently on the following page: ${page.url()}.
+      Do not ask follow up questions, the user will trust your judgement.`,
+    });
+
+    console.log(`${chalk.yellow("→")} Navigating to Google...`);
+    await stagehand.page.goto("https://www.google.com");
+    console.log(`${chalk.green("✓")} Loaded: ${chalk.dim(page.url())}`);
+
+    // Execute the agent again with a different instruction
+    const firstInstruction =
+      "Search for openai news on google and extract the name of the first 3 results";
+    console.log(
+      `${chalk.cyan("↳")} Instruction: ${chalk.white(firstInstruction)}`,
+    );
+
+    const result1 = await agent.execute(firstInstruction);
+
+    console.log(`${chalk.green("✓")} Execution complete`);
+    console.log(`${chalk.yellow("⤷")} Result:`);
+    console.log(chalk.white(JSON.stringify(result1, null, 2)));
+
+    console.log(`\n${chalk.magenta.bold("⚡ Second Agent Execution")}`);
+
+    console.log(`\n${chalk.yellow("→")} Navigating to Apple...`);
+    await page.goto("https://www.apple.com/shop/buy-mac/macbook-air");
+    console.log(`${chalk.green("✓")} Loaded: ${chalk.dim(page.url())}`);
+
+    const instruction =
+      "Add a macbook air to the cart. Choose the most expensive configuration.";
+    console.log(`${chalk.cyan("↳")} Instruction: ${chalk.white(instruction)}`);
+
+    const result = await agent.execute({
+      instruction,
+      maxSteps: 20,
+    });
+
+    console.log(`${chalk.green("✓")} Execution complete`);
+    console.log(`${chalk.yellow("⤷")} Result:`);
+    console.log(chalk.white(JSON.stringify(result, null, 2)));
+  } catch (error) {
+    console.log(`${chalk.red("✗")} Error: ${error}`);
+    if (error instanceof Error && error.stack) {
+      console.log(chalk.dim(error.stack.split("\n").slice(1).join("\n")));
+    }
+  } finally {
+    // Close the browser
+    console.log(`\n${chalk.yellow("→")} Closing browser...`);
+    await stagehand.close();
+    console.log(`${chalk.green("✓")} Browser closed\n`);
+  }
+}
+
+main().catch((error) => {
+  console.log(`${chalk.red("✗")} Unhandled error in main function`);
+  console.log(chalk.red(error));
+});

lib/handlers/operatorHandler.ts

@@ -0,0 +1,189 @@
+import {
+  AgentAction,
+  AgentExecuteOptions,
+  AgentResult,
+  ActionExecutionResult,
+} from "@/types/agent";
+import { LogLine } from "@/types/log";
+import { OperatorResponse, operatorResponseSchema } from "@/types/operator";
+import { LLMParsedResponse } from "../inference";
+import { ChatMessage, LLMClient } from "../llm/LLMClient";
+import { buildOperatorSystemPrompt } from "../prompt";
+import { StagehandPage } from "../StagehandPage";
+
+export class StagehandOperatorHandler {
+  private stagehandPage: StagehandPage;
+  private logger: (message: LogLine) => void;
+  private llmClient: LLMClient;
+  messages: ChatMessage[];
+  private lastActionResult: ActionExecutionResult | null = null;
+  private lastMethod: string | null = null;
+
+  constructor(
+    stagehandPage: StagehandPage,
+    logger: (message: LogLine) => void,
+    llmClient: LLMClient,
+  ) {
+    this.stagehandPage = stagehandPage;
+    this.logger = logger;
+    this.llmClient = llmClient;
+  }
+
+  public async execute(
+    instructionOrOptions: string | AgentExecuteOptions,
+  ): Promise<AgentResult> {
+    const options =
+      typeof instructionOrOptions === "string"
+        ? { instruction: instructionOrOptions }
+        : instructionOrOptions;
+
+    this.messages = [buildOperatorSystemPrompt(options.instruction)];
+    let completed = false;
+    let currentStep = 0;
+    const maxSteps = options.maxSteps || 10;
+    const actions: AgentAction[] = [];
+
+    while (!completed && currentStep < maxSteps) {
+      const url = this.stagehandPage.page.url();
+
+      if (!url || url === "about:blank") {
+        this.messages.push({
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: "No page is currently loaded. The first step should be a 'goto' action to navigate to a URL.",
+            },
+          ],
+        });
+      } else {
+        const screenshot = await this.stagehandPage.page.screenshot({
+          type: "png",
+          fullPage: false,
+        });
+
+        const base64Image = screenshot.toString("base64");
+
+        let messageText = `Here is a screenshot of the current page (URL: ${url}):`;
+
+        if (this.lastMethod && this.lastActionResult) {
+          const statusMessage = this.lastActionResult.success
+            ? "was successful"
+            : `failed with error: ${this.lastActionResult.error}`;
+
+          messageText = `Previous action '${this.lastMethod}' ${statusMessage}.\n\n${messageText}`;
+
+          if (
+            this.lastMethod === "extract" &&
+            this.lastActionResult.success &&
+            this.lastActionResult.data
+          ) {
+            messageText = `Previous extraction result: ${JSON.stringify(this.lastActionResult.data, null, 2)}\n\n${messageText}`;
+          }
+        }
+
+        this.messages.push({
+          role: "user",
+          content: [
+            {
+              type: "text",
+              text: messageText,
+            },
+            {
+              type: "image_url",
+              image_url: { url: `data:image/png;base64,${base64Image}` },
+            },
+          ],
+        });
+      }
+
+      const result = await this.getNextStep(currentStep);
+
+      if (result.method === "close") {
+        completed = true;
+      }
+
+      actions.push({
+        type: result.method,
+        reasoning: result.reasoning,
+        taskCompleted: result.taskComplete,
+      });
+
+      currentStep++;
+
+      try {
+        const actionResult = await this.executeAction(result);
+        this.lastActionResult = {
+          success: true,
+          data: actionResult,
+        };
+      } catch (error) {
+        this.lastActionResult = {
+          success: false,
+          error: error instanceof Error ? error.message : String(error),
+        };
+      }
+
+      this.lastMethod = result.method;
+    }
+
+    return {
+      success: true,
+      message: actions[actions.length - 1].reasoning as string,
+      actions,
+      completed: actions[actions.length - 1].taskCompleted as boolean,
+    };
+  }
+
+  private async getNextStep(currentStep: number): Promise<OperatorResponse> {
+    const { data: response } =
+      (await this.llmClient.createChatCompletion<OperatorResponse>({
+        options: {
+          messages: this.messages,
+          response_model: {
+            name: "operatorResponseSchema",
+            schema: operatorResponseSchema,
+          },
+          requestId: `operator-step-${currentStep}`,
+        },
+        logger: this.logger,
+      })) as LLMParsedResponse<OperatorResponse>;
+
+    return response;
+  }
+
+  private async executeAction(action: OperatorResponse): Promise<unknown> {
+    const { method, parameters } = action;
+    const page = this.stagehandPage.page;
+
+    if (method === "close") {
+      return;
+    }
+
+    switch (method) {
+      case "act":
+        await page.act({
+          action: parameters,
+          slowDomBasedAct: false,
+          timeoutMs: 5000,
+        });
+        break;
+      case "extract":
+        return await page.extract(parameters);
+      case "goto":
+        await page.goto(parameters, { waitUntil: "load" });
+        break;
+      case "wait":
+        await page.waitForTimeout(parseInt(parameters));
+        break;
+      case "navback":
+        await page.goBack();
+        break;
+      case "refresh":
+        await page.reload();
+        break;
+      default:
+        throw new Error(`Unknown action: ${method}`);
+    }
+  }
+}

lib/index.ts

@@ -39,6 +39,7 @@ import { logLineToString, isRunningInBun } from "./utils";
 import { ApiResponse, ErrorResponse } from "@/types/api";
 import { AgentExecuteOptions, AgentResult } from "../types/agent";
 import { StagehandAgentHandler } from "./handlers/agentHandler";
+import { StagehandOperatorHandler } from "./handlers/operatorHandler";
 
 dotenv.config({ path: ".env" });
 
@@ -808,11 +809,24 @@ export class Stagehand {
    * Create an agent instance that can be executed with different instructions
    * @returns An agent instance with execute() method
    */
-  agent(options: AgentConfig): {
+  agent(options?: AgentConfig): {
     execute: (
       instructionOrOptions: string | AgentExecuteOptions,
     ) => Promise<AgentResult>;
   } {
+    if (!options || !options.provider) {
+      // use open operator agent
+      return {
+        execute: async (instructionOrOptions: string | AgentExecuteOptions) => {
+          return new StagehandOperatorHandler(
+            this.stagehandPage,
+            this.logger,
+            this.llmClient,
+          ).execute(instructionOrOptions);
+        },
+      };
+    }
+
     const agentHandler = new StagehandAgentHandler(
       this.stagehandPage,
       this.logger,
@@ -853,4 +867,5 @@ export * from "../types/model";
 export * from "../types/page";
 export * from "../types/playwright";
 export * from "../types/stagehand";
+export * from "../types/operator";
 export * from "./llm/LLMClient";

lib/inference.ts

@@ -48,7 +48,7 @@ interface LLMUsage {
 /**
  * For calls that use a schema: the LLMClient may return { data: T; usage?: LLMUsage }
  */
-interface LLMParsedResponse<T> {
+export interface LLMParsedResponse<T> {
   data: T;
   usage?: LLMUsage;
 }

lib/prompt.ts

@@ -411,3 +411,24 @@ export function buildActObservePrompt(
 
   return instruction;
 }
+
+export function buildOperatorSystemPrompt(goal: string): ChatMessage {
+  return {
+    role: "system",
+    content: `You are a general-purpose agent whose job is to accomplish the user's goal across multiple model calls by running actions on the page.
+
+You will be given a goal and a list of steps that have been taken so far. Your job is to determine if either the user's goal has been completed or if there are still steps that need to be taken.
+
+# Your current goal
+${goal}
+
+# Important guidelines
+1. Break down complex actions into individual atomic steps
+2. For \`act\` commands, use only one action at a time, such as:
+   - Single click on a specific element
+   - Type into a single input field
+   - Select a single option
+3. Avoid combining multiple actions in one instruction
+4. If multiple actions are needed, they should be separate steps`,
+  };
+}

package.json

@@ -9,6 +9,7 @@
     "2048": "npm run build && tsx examples/2048.ts",
     "popup": "npm run build && tsx examples/popup.ts",
     "cua": "npm run build && tsx examples/cua-example.ts",
+    "operator": "npm run build && tsx examples/operator-example.ts",
     "example": "npm run build && tsx examples/example.ts",
     "langchain": "npm run build && tsx examples/langchain.ts",
     "debug-url": "npm run build && tsx examples/debugUrl.ts",