diff --git a/cli/templates/files/ai-agent/evals/assistant.eval.ts b/cli/templates/files/ai-agent/evals/assistant.eval.ts index ac6a37ace0..fade362257 100644 --- a/cli/templates/files/ai-agent/evals/assistant.eval.ts +++ b/cli/templates/files/ai-agent/evals/assistant.eval.ts @@ -1,34 +1,43 @@ import { datasets, evalAgent, judges, metrics } from "veryfront/eval"; +// An eval runs your agent against fixed inputs and grades the results. +// This one asks the assistant a single arithmetic question, then checks that it +// used the calculator, that no tool errored, and that the answer is correct. +// +// Run it with: npm run eval -- assistant export default evalAgent({ name: "Assistant smoke test", target: "agent:assistant", + + // The questions to ask. `reference` is the answer you expect; the judge below + // grades the agent's answer against it. dataset: datasets.inline([ { id: "calculator", input: "Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.", - reference: "$99.71 total; two people pay $33.24 and one pays $33.23.", + reference: + "The tip is $15.21 and the total is $99.71. Two people pay $33.24 and one pays $33.23.", }, ]), + + // Each metric is a gate: if any one fails, the eval fails. metrics: [ - metrics.answer.regex({ - pattern: String.raw`(? { ]); }); - it("accepts sentence punctuation without accepting longer monetary values", async () => { + it("grades the starter's money answer with a rubric a reader can follow", async () => { + // The starter eval is the first eval most people ever read, so it has to be + // legible. It used to gate each amount with a hand-rolled lookaround regex + // (`(? metric.name), [ - "answer.regex", - "answer.regex", - "answer.regex", - "answer.regex", + + assertEquals(assistantEval.metrics.map((metric) => metric.name), [ + "agent.calledTool", + "agent.noFailedTools", + "judge.rubric", ]); - const createRecord = (text: string): EvalRecord => ({ - id: "calculator:1", - evalId: "eval:assistant", - exampleId: "calculator", - repetition: 1, - input: "Calculate the tip and split.", - output: { text }, - reference: "$99.71 total; two people pay $33.24 and one pays $33.23.", - metadata: {}, - trace: { events: [], toolCalls: [] }, - usage: {}, - durationMs: 1, - completed: true, - }); - const validResults = await Promise.all( - moneyMetrics.map((metric) => - metric.evaluate( - createRecord( - "The tip is $15.21. The total is $99.71. Two people pay $33.24, and one pays $33.23.", - ), - ) - ), + const source = await Deno.readTextFile( + new URL("./files/ai-agent/evals/assistant.eval.ts", import.meta.url), + ); + assertEquals( + source.includes("metrics.answer.regex"), + false, + "the starter eval should not teach hand-rolled regex assertions", + ); + assertEquals( + source.includes("String.raw"), + false, + "the starter eval should not need raw strings to express an assertion", ); - assertEquals(validResults.map((result) => result.pass), [true, true, true, true]); - const tipMetric = moneyMetrics[0]; - assertExists(tipMetric); - for (const valid of ["$15.21.", String.raw`\$15.21`, "($15.21)", "**$15.21**"]) { - assertEquals((await tipMetric.evaluate(createRecord(valid))).pass, true); + // Dropping the regexes moved exactness onto the judge, so the rubric has to + // spell out both the amounts and that near-misses fail. + const rubricMetric = assistantEval.metrics.at(-1); + assertExists(rubricMetric); + const rubric = String(rubricMetric.config?.rubric ?? ""); + for (const amount of ["$15.21", "$99.71", "$33.24", "$33.23"]) { + assertEquals( + rubric.includes(amount), + true, + `the rubric should name the expected ${amount}`, + ); } - for ( - const invalid of [ - "-15.21", - "-$15.21", - String.raw`-\$15.21`, - "115.21", - "$15.210", - "$15.21.0", - ] - ) { - assertEquals((await tipMetric.evaluate(createRecord(invalid))).pass, false); + assertEquals( + /exact/i.test(rubric), + true, + "the rubric should require exact amounts now that no regex enforces it", + ); + for (const nearMiss of ["$33.2366", "$133.23"]) { + assertEquals( + rubric.includes(nearMiss), + true, + `the rubric should show ${nearMiss} as a failing near-miss`, + ); } }); diff --git a/cli/templates/manifest.json b/cli/templates/manifest.json index 7559281641..ba55bfdf5a 100644 --- a/cli/templates/manifest.json +++ b/cli/templates/manifest.json @@ -27,7 +27,7 @@ "app/layout.tsx": "import \"../globals.css\";\nimport { Head } from \"veryfront/head\";\n\nexport default function RootLayout({\n children,\n}: {\n children: React.ReactNode;\n}): React.ReactNode {\n return (\n <>\n
\n