From 4eaf8e4689dc5a8981cc4bea3b82e254468cb977 Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Sun, 1 Nov 2026 00:59:29 +0200 Subject: [PATCH 1/2] fix(evals): GPT eval summaries no longer call a proxied run free Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/run.test.ts | 21 +++++++++++++++++++++ evals/mcp-tool-use/run.ts | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 33edb6465ab4c..222ad132d7970 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -1784,6 +1784,27 @@ it("OWN-03 passes a reported conflict and fails reuse or close of the owner's se assert.isNotEmpty(failed(c.expect, trace("", "Saved /tmp/proof/settings.png."))); }); +it("never calls a proxied GPT run free", () => { + const usage = { input: 10, output: 0, cacheWrite: 0, cacheRead: 0 }; + const row = { + id: "A", + kind: "bad" as const, + pattern: "p", + role: "top", + pass: true, + usage, + shellUsage: usage, + }; + const summary = summarize([row], { + header: "run", + shellModel: "claude-haiku-5-5", + model: "gpt-6.1-sol", + }); + assert.include(summary, "the agent is not priced"); + assert.include(summary, "bill an API key"); + assert.notInclude(summary, "not applicable"); +}); + it("leaves errored samples out of pass rates and lists them in the summary", () => { const usage = { input: 10, output: 0, cacheWrite: 0, cacheRead: 0 }; const row = (id: string, pass: boolean, error?: string) => ({ diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 7cad56ae98048..31b587b42eb8e 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -1834,7 +1834,7 @@ export const summarize = ( `- Pass: ${percent(passRate(rows))} of ${finished} samples.`, ...(errors.length > 0 ? [errorNote] : []), codex - ? "- Cost: not applicable (subscription through proxy)." + ? "- Cost: the agent is not priced. `vibeproxy` in `~/.codex/config.toml` may use a subscription or bill an API key per token; agent tokens are below." : `- Estimated cost $${cost.toFixed(2)}, $${(cost / rows.length).toFixed(3)} per sample.`, `- Agent tokens: ${agent.tokens}${codex ? "." : ` ($${agent.cost.toFixed(2)}).`}`, `- Shell simulator (${options.shellModel}) tokens: ${shell.tokens} (API-price estimate $${shell.cost.toFixed(2)}).`, From c0a65526e88fc1bd739cd6b6923cb67f36b0b7da Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Sun, 1 Nov 2026 01:07:00 +0200 Subject: [PATCH 2/2] test(evals): drop the copy-text test docs/internals/testing.md leaves copy text out of tests; Macroscope flagged it. Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/run.test.ts | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 222ad132d7970..33edb6465ab4c 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -1784,27 +1784,6 @@ it("OWN-03 passes a reported conflict and fails reuse or close of the owner's se assert.isNotEmpty(failed(c.expect, trace("", "Saved /tmp/proof/settings.png."))); }); -it("never calls a proxied GPT run free", () => { - const usage = { input: 10, output: 0, cacheWrite: 0, cacheRead: 0 }; - const row = { - id: "A", - kind: "bad" as const, - pattern: "p", - role: "top", - pass: true, - usage, - shellUsage: usage, - }; - const summary = summarize([row], { - header: "run", - shellModel: "claude-haiku-5-5", - model: "gpt-6.1-sol", - }); - assert.include(summary, "the agent is not priced"); - assert.include(summary, "bill an API key"); - assert.notInclude(summary, "not applicable"); -}); - it("leaves errored samples out of pass rates and lists them in the summary", () => { const usage = { input: 10, output: 0, cacheWrite: 0, cacheRead: 0 }; const row = (id: string, pass: boolean, error?: string) => ({