feat(evals): replay GOAT tool-use mistakes (chase 261028 B19)

docs/operations/agent-surface-eval.md

@@ -164,6 +164,65 @@ The manual tasks need connected hosts and a person who approves or finishes sign
 runner includes them only with `--include-manual`. D8's historical provider-sync
 verdict requires those tasks; this does not gate `goat_screen_run`.
 
+## Replaying tool-use mistakes
+
+The separate [replay cases](../../scripts/agent-surface/eval/replay-cases.ts) cover review
+rounds, async delegation, managed worktrees, PR ownership, thread links, and HTML preview.
+They use invented histories adapted from upstream's tool-use regressions. They do not change
+the paired campaign task set or count toward its gate.
+
+Export a case with this checkout's GOAT instructions and the tools its thread listing serves:
+
+```sh
+node scripts/agent-surface/eval/replay-run.ts --case async-delegate-ends-turn --listing core --context
+```
+
+Repeat with `--listing full` to compare the two surfaces. Context generation uses the existing
+inert catalog fixture; no installed GOAT server, credentials, provider, or captured harness
+prompt is needed. The output is harness-independent GOAT context, not a complete Claude or
+Codex request. An external harness supplies its native tools, honors the case's role and prior
+turns, and records the new continuation. This command does not run that harness or simulate
+future tool results.
+
+Save the continuation as JSON with `version`, `caseId`, and `listing` from the exported context,
+plus `calls`, `text`, and `stopReason`. Calls contain `tool`, object `input`, and optional
+`isError` (set it to `true` for rejected calls). Record executed nested code-mode calls in order;
+the grader does not execute or inspect JavaScript source. Include only the new continuation,
+because setup calls cannot satisfy the case's requirements. For example:
+
+```json
+{
+  "version": 3,
+  "caseId": "async-delegate-ends-turn",
+  "listing": "core",
+  "calls": [
+    {
+      "tool": "mcp__goat__goat_call",
+      "input": {
+        "path": "task.create",
+        "args": { "task": "Compare the cache designs", "mode": "async" }
+      }
+    }
+  ],
+  "text": "The research task is running; its result will arrive automatically.",
+  "stopReason": "end_turn"
+}
+```
+
+Grade it without calling a model or executing any replayed action:
+
+```sh
+node scripts/agent-surface/eval/replay-run.ts --case async-delegate-ends-turn --listing core --trace ../continuation.json
+```
+
+Exit 1 means a rule failed. `max_steps` and `error` stop reasons always fail. The grader checks
+required and forbidden calls, partial arguments, ordering, call counts, shell patterns and
+answer text. It resolves GOAT names through the current registry, accepts `goat_call` dispatch
+in either listing, and rejects unknown paths or direct calls outside the selected listing.
+These are behavioral checks on the supplied trace, not proof that its arguments passed the
+server's schemas or that the requested work happened. Keep real transcripts and credentials
+out of committed replay data. Changing a case or grading rule raises `REPLAY_VERSION`.
+
 ## Not yet possible
 
 - `goat_eval` is unbuilt. Its candidate arm remains historical; this runner does

knip.jsonc

@@ -25,6 +25,7 @@
         // These fixture scripts are injected into a browser page by path.
         "fixtures/*/*.js",
         "knip-schemas.ts",
+        "agent-surface/eval/replay-run.ts",
         "mobile-native-client.ts",
         "setup-worktree.ts",
         "worktree-add.ts",

scripts/agent-surface/eval/replay-cases.ts

@@ -0,0 +1,151 @@
+import type { ReplayCase } from "./replay.ts";
+
+export const REPLAY_VERSION = 3;
+const pr = "https://github.com/example/replay-fixture/pull/124";
+const threadId = "11111111-1111-4111-8111-111111111111";
+
+export const replayCases: ReadonlyArray<ReplayCase> = [
+  {
+    id: "review-round-creates-new-task",
+    upstreamId: "review-round-uses-thread-send",
+    role: "top",
+    experience:
+      "A second review round continued the old child thread and lost its task relationship.",
+    priorTurns: [
+      {
+        role: "user",
+        text: "Get a delegated review of the cache migration. Repeat after fixing findings.",
+      },
+      {
+        role: "assistant",
+        text: "Round one found that the cache migration discards version-four entries. That is now fixed.",
+        calls: [
+          {
+            tool: "goat_task_read",
+            input: { taskId: "review-1" },
+            result: {
+              taskId: "review-1",
+              childThreadId: threadId,
+              status: "completed",
+              summary: "The cache migration discards version-four entries.",
+            },
+          },
+        ],
+      },
+    ],
+    prompt:
+      "Run round two with a new delegated review task. Include the cache finding and the fix.",
+    expect: {
+      calls: [{ tool: "task.create", args: { role: "review", task: "~cache" } }],
+      notCalls: ["thread.send"],
+    },
+  },
+  {
+    id: "async-delegate-ends-turn",
+    upstreamId: "async-delegate-then-end-turn",
+    role: "top",
+    experience: "An agent polled after starting async work even though GOAT delivers completion.",
+    priorTurns: [],
+    prompt:
+      "Start a GOAT delegated research task to compare the two cache designs, then wait for its automatic delivery.",
+    expect: {
+      calls: [{ tool: "task.create", args: { mode: "async" } }],
+      notCalls: ["task.read", "thread.wait"],
+      endsTurnAfter: "task.create",
+      shellMustNot: ["\\bsleep\\b", "inotifywait|fswatch"],
+    },
+  },
+  {
+    id: "delegate-uses-managed-worktree",
+    upstreamId: "worktree-add-for-parallel-pr-delegates",
+    role: "top",
+    experience: "A parent created an unmanaged worktree for a child implementation lane.",
+    priorTurns: [],
+    prompt:
+      "Delegate the cache fix to a GOAT child with its own isolated checkout. GOAT should manage the checkout.",
+    expect: {
+      calls: [{ tool: "task.create", args: { isolation: "worktree" } }],
+      callsMustNot: [
+        { tool: "task.create", args: { task: "~git worktree add" } },
+        { tool: "task.create", args: { task: "~git clone" } },
+      ],
+      notCalls: ["thread.launch"],
+      shellMustNot: ["git\\s+worktree\\s+add", "git\\s+clone"],
+    },
+  },
+  {
+    id: "child-links-without-watching",
+    upstreamId: "child-pushes-existing-pr-links",
+    role: "delegated-child",
+    experience:
+      "A delegated implementation child watched checks instead of reporting to its parent.",
+    priorTurns: [
+      {
+        role: "assistant",
+        text: `The authorized fix was pushed to ${pr}, head abcdef1234567890. No checks have run yet.`,
+      },
+    ],
+    prompt:
+      "Link the PR you pushed and report its URL, head and check state to your parent. Do not wait for CI.",
+    expect: {
+      calls: [{ tool: "pull_request.link", args: { url: pr } }],
+      notCalls: ["pull_request.watch"],
+      shellMustNot: [
+        "\\bsleep\\b",
+        "gh pr checks[^\\n]*--watch",
+        "gh run watch",
+        "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b",
+        "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b",
+        "pr-wait",
+      ],
+      textMust: ["https://github\\.com/example/replay-fixture/pull/124", "abcdef1234567890"],
+    },
+  },
+  {
+    id: "review-only-does-not-link-pr",
+    upstreamId: "review-child-links-reviewed-pr",
+    role: "delegated-child",
+    experience: "A reviewer linked a PR it only read, making the thread appear to own that work.",
+    priorTurns: [
+      {
+        role: "assistant",
+        text: `I read the diff of ${pr}. The cache key includes the version and the migration preserves entries. No findings.`,
+      },
+    ],
+    prompt:
+      "Finish your read-only review with a verdict. Do not push or take ownership of this PR.",
+    expect: {
+      notCalls: ["pull_request.link", "pull_request.watch", "pull_request.list"],
+      shellMustNot: ["git\\s+push", "gh pr (create|merge)"],
+      textMust: ["[Nn]o findings|PASS"],
+    },
+  },
+  {
+    id: "thread-read-returns-link",
+    upstreamId: "thread-mention-without-link-after-read",
+    role: "top",
+    experience: "An agent mentioned a thread it read without a clickable GOAT thread link.",
+    priorTurns: [],
+    prompt: `Read the latest messages of thread ${threadId}, summarize them, and link the thread. Read at most twenty items.`,
+    expect: {
+      calls: [{ tool: "thread.read", args: { threadId, limit: "<=20" } }],
+      textMust: [`goat-thread://v1/${threadId}`],
+    },
+  },
+  {
+    id: "preview-before-render",
+    upstreamId: "render-without-html-preview",
+    role: "top",
+    experience:
+      "An agent published an inline visualization without first checking the HTML preview.",
+    priorTurns: [],
+    prompt:
+      "Make an inline HTML diagram of the cache flow. Check the preview before publishing it in this conversation.",
+    expect: {
+      calls: [{ tool: "html.preview" }, { tool: "html.render" }],
+      order: ["html.preview", "html.render"],
+      maxCalls: { "html.render": 1 },
+      shellMustNot: ["http\\.server", "npx (serve|http-server)"],
+    },
+  },
+];

scripts/agent-surface/eval/replay-run.ts

@@ -0,0 +1,88 @@
+// @effect-diagnostics nodeBuiltinImport:off globalConsole:off - a plain Node script that reads offline replay context and trace files; no Effect runtime.
+import * as NodeFS from "node:fs";
+import * as NodeUtil from "node:util";
+import { loadServedCatalog } from "../servedCatalog.ts";
+import { REPLAY_VERSION, replayCases } from "./replay-cases.ts";
+import { decodeReplayRun, gradeReplay } from "./replay.ts";
+
+const { values } = NodeUtil.parseArgs({
+  options: {
+    case: { type: "string" },
+    listing: { type: "string", default: "core" },
+    context: { type: "boolean", default: false },
+    trace: { type: "string" },
+  },
+});
+const testCase = replayCases.find((entry) => entry.id === values.case);
+if (!testCase)
+  throw new Error("--case must name one of: " + replayCases.map((entry) => entry.id).join(", "));
+const listing = values.listing;
+if (listing !== "core" && listing !== "full") throw new Error("--listing must be core or full");
+if (values.context === Boolean(values.trace))
+  throw new Error("Choose --context or --trace <file.json>");
+
+const catalog = await loadServedCatalog();
+try {
+  const tools = await catalog.listedAsThread(listing);
+  if (values.context) {
+    // Loaded at runtime: the scripts project does not compile provider-core.
+    const instructions = (await import(
+      new URL(
+        "../../../packages/provider-core/src/server/orchestrationInstructions.ts",
+        import.meta.url,
+      ).href
+    )) as {
+      readonly goatOrchestrationSystemPrompt: (hasGoatMcp: boolean) => string;
+      readonly GOAT_BROWSER_TOOL_INSTRUCTIONS: string;
+    };
+    console.log(
+      JSON.stringify(
+        {
+          version: REPLAY_VERSION,
+          caseId: testCase.id,
+          listing,
+          role: testCase.role,
+          instructions:
+            instructions.goatOrchestrationSystemPrompt(true) +
+            instructions.GOAT_BROWSER_TOOL_INSTRUCTIONS,
+          tools,
+          priorTurns: testCase.priorTurns,
+          prompt: testCase.prompt,
+        },
+        null,
+        2,
+      ),
+    );
+  } else {
+    const tracePath = values.trace;
+    if (!tracePath) throw new Error("--trace needs a file path");
+    const raw: unknown = JSON.parse(NodeFS.readFileSync(tracePath, "utf8"));
+    if (
+      typeof raw !== "object" ||
+      raw === null ||
+      !("version" in raw) ||
+      raw.version !== REPLAY_VERSION ||
+      !("caseId" in raw) ||
+      raw.caseId !== testCase.id ||
+      !("listing" in raw) ||
+      raw.listing !== listing
+    )
+      throw new Error("Trace version, caseId and listing must match the replay context");
+    const checks = gradeReplay(
+      testCase.expect,
+      decodeReplayRun(raw),
+      tools.map((tool) => tool.name),
+    );
+    const pass = checks.every((check) => check.pass);
+    console.log(
+      JSON.stringify(
+        { version: REPLAY_VERSION, caseId: testCase.id, listing, pass, checks },
+        null,
+        2,
+      ),
+    );
+    if (!pass) process.exitCode = 1;
+  }
+} finally {
+  await catalog.dispose();
+}