From 811a70eebb03879fca8357f697fc8122154b06e0 Mon Sep 17 00:00:00 2001
From: Kentaro Wakayama <kentaro@codersociety.com>
Date: Thu, 6 Aug 2026 13:15:39 +0200
Subject: [PATCH] chore(templates): make the starter eval readable

The ai-agent starter eval is the first eval most people ever read. It opened
with four hand-rolled lookaround regexes:

    metrics.answer.regex({
      pattern: String.raw`(?<![-\d.\\])\\?\$33\.23(?!\d|\.\d)`,
    }).gate(),

Those are exact, and they are exact for good reasons earned the hard way:
#3162 replaced `contains({ text: "33.23" })` because it passed on 33.2366, and
a follow-up commit added the `\\?` and the backslash in the lookbehind because
Markdown-escaped output let the engine restart at the dollar token and skip a
preceding minus.

But a starter file teaches by example, and this one taught that asserting an
agent said "$33.23" takes a negative lookbehind. Nobody reading it learns what
the eval checks; they learn to copy a regex they cannot modify.

The exactness requirement moves into the rubric judge, in prose:

    "Every amount must be exact to the cent: $33.2366 and $133.23 are wrong."

That is the same constraint, stated so a reader can check it and change it.

This is a deliberate trade. #3162's directive was to keep deterministic
currency gates alongside the rubric judge, and this branch does not: monetary
correctness is now graded by a model, so it costs an LLM call and can vary run
to run. The gate threshold stays at 0.8 and the tool gates
(`calledTool("calculator")`, `noFailedTools()`) stay deterministic, so a run
that skips the calculator or errors a tool still fails without the judge.

The regression worth guarding is the file drifting back toward unreadable, so
the template test asserts the eval carries no `metrics.answer.regex` and no
`String.raw`, and -- because nothing else enforces exactness now -- that the
rubric still names all four amounts, still says "exact", and still shows the
near-misses it has to reject.

Tested: template + eval suites (23 files, 217 steps); fmt; lint; typecheck;
manifest check. Mutation-checked the new guard by dropping the exactness
sentence from the rubric and confirming the test fails.

Not-tested: no live eval run against a model; the rubric judge's ability to
reject $33.2366 in practice is asserted in prose, not measured.
---
 .../files/ai-agent/evals/assistant.eval.ts    | 39 +++++---
 cli/templates/index.test.ts                   | 92 +++++++++----------
 cli/templates/manifest.json                   |  2 +-
 3 files changed, 71 insertions(+), 62 deletions(-)

diff --git a/cli/templates/files/ai-agent/evals/assistant.eval.ts b/cli/templates/files/ai-agent/evals/assistant.eval.ts
index ac6a37ace0..fade362257 100644
--- a/cli/templates/files/ai-agent/evals/assistant.eval.ts
+++ b/cli/templates/files/ai-agent/evals/assistant.eval.ts
@@ -1,34 +1,43 @@
 import { datasets, evalAgent, judges, metrics } from "veryfront/eval";
 
+// An eval runs your agent against fixed inputs and grades the results.
+// This one asks the assistant a single arithmetic question, then checks that it
+// used the calculator, that no tool errored, and that the answer is correct.
+//
+// Run it with: npm run eval -- assistant
 export default evalAgent({
   name: "Assistant smoke test",
   target: "agent:assistant",
+
+  // The questions to ask. `reference` is the answer you expect; the judge below
+  // grades the agent's answer against it.
   dataset: datasets.inline([
     {
       id: "calculator",
       input:
         "Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.",
-      reference: "$99.71 total; two people pay $33.24 and one pays $33.23.",
+      reference:
+        "The tip is $15.21 and the total is $99.71. Two people pay $33.24 and one pays $33.23.",
     },
   ]),
+
+  // Each metric is a gate: if any one fails, the eval fails.
   metrics: [
-    metrics.answer.regex({
-      pattern: String.raw`(?<![-\d.\\])\\?\$15\.21(?!\d|\.\d)`,
-    }).gate(),
-    metrics.answer.regex({
-      pattern: String.raw`(?<![-\d.\\])\\?\$99\.71(?!\d|\.\d)`,
-    }).gate(),
-    metrics.answer.regex({
-      pattern: String.raw`(?<![-\d.\\])\\?\$33\.24(?!\d|\.\d)`,
-    }).gate(),
-    metrics.answer.regex({
-      pattern: String.raw`(?<![-\d.\\])\\?\$33\.23(?!\d|\.\d)`,
-    }).gate(),
+    // The agent must do the arithmetic with the calculator tool, not in its head.
     metrics.agent.calledTool("calculator").gate(),
+
+    // No tool call may error.
     metrics.agent.noFailedTools().gate(),
+
+    // A second model reads the answer and scores it from 0 to 1 against this
+    // rubric. It needs at least 0.8 to pass.
     metrics.judge.rubric({
-      rubric:
-        "The answer must correctly state the $15.21 tip, $99.71 total, and a cent-exact split of two payments of $33.24 and one of $33.23. It should explain the result briefly.",
+      rubric: [
+        "The answer must state a tip of $15.21 and a total of $99.71.",
+        "It must split the total into $33.24, $33.24, and $33.23.",
+        "Every amount must be exact to the cent: $33.2366 and $133.23 are wrong.",
+        "The explanation must be brief.",
+      ].join(" "),
       judge: judges.llm.rubric(),
     }).gate({ min: 0.8 }),
   ],
diff --git a/cli/templates/index.test.ts b/cli/templates/index.test.ts
index 9915068d64..b72b3a2209 100644
--- a/cli/templates/index.test.ts
+++ b/cli/templates/index.test.ts
@@ -7,7 +7,6 @@ import {
 } from "#veryfront/testing/assert.ts";
 import { describe, it } from "#veryfront/testing/bdd.ts";
 import { fromFileUrl } from "#veryfront/compat/path";
-import type { EvalRecord } from "veryfront/eval";
 
 import { getTemplate, getTemplateConfig, templateConfigs } from "./index.ts";
 import { STARTER_TEMPLATE_NAMES, type TemplateName } from "./types.ts";
@@ -179,59 +178,60 @@ describe("cli/templates", () => {
     ]);
   });
 
-  it("accepts sentence punctuation without accepting longer monetary values", async () => {
+  it("grades the starter's money answer with a rubric a reader can follow", async () => {
+    // The starter eval is the first eval most people ever read, so it has to be
+    // legible. It used to gate each amount with a hand-rolled lookaround regex
+    // (`(?<![-\d.\\])\\?\$33\.23(?!\d|\.\d)`) to reject near-misses like
+    // $33.2366 and $133.23. That was exact but unreadable, and copy-pasting it
+    // is the wrong lesson to teach. The rubric judge now carries the exactness
+    // requirement in prose instead.
     const { default: assistantEval } = await import(
       "./files/ai-agent/evals/assistant.eval.ts"
     );
-    const moneyMetrics = assistantEval.metrics.slice(0, 4);
-    assertEquals(moneyMetrics.map((metric) => metric.name), [
-      "answer.regex",
-      "answer.regex",
-      "answer.regex",
-      "answer.regex",
+
+    assertEquals(assistantEval.metrics.map((metric) => metric.name), [
+      "agent.calledTool",
+      "agent.noFailedTools",
+      "judge.rubric",
     ]);
-    const createRecord = (text: string): EvalRecord => ({
-      id: "calculator:1",
-      evalId: "eval:assistant",
-      exampleId: "calculator",
-      repetition: 1,
-      input: "Calculate the tip and split.",
-      output: { text },
-      reference: "$99.71 total; two people pay $33.24 and one pays $33.23.",
-      metadata: {},
-      trace: { events: [], toolCalls: [] },
-      usage: {},
-      durationMs: 1,
-      completed: true,
-    });
 
-    const validResults = await Promise.all(
-      moneyMetrics.map((metric) =>
-        metric.evaluate(
-          createRecord(
-            "The tip is $15.21. The total is $99.71. Two people pay $33.24, and one pays $33.23.",
-          ),
-        )
-      ),
+    const source = await Deno.readTextFile(
+      new URL("./files/ai-agent/evals/assistant.eval.ts", import.meta.url),
+    );
+    assertEquals(
+      source.includes("metrics.answer.regex"),
+      false,
+      "the starter eval should not teach hand-rolled regex assertions",
+    );
+    assertEquals(
+      source.includes("String.raw"),
+      false,
+      "the starter eval should not need raw strings to express an assertion",
     );
-    assertEquals(validResults.map((result) => result.pass), [true, true, true, true]);
 
-    const tipMetric = moneyMetrics[0];
-    assertExists(tipMetric);
-    for (const valid of ["$15.21.", String.raw`\$15.21`, "($15.21)", "**$15.21**"]) {
-      assertEquals((await tipMetric.evaluate(createRecord(valid))).pass, true);
+    // Dropping the regexes moved exactness onto the judge, so the rubric has to
+    // spell out both the amounts and that near-misses fail.
+    const rubricMetric = assistantEval.metrics.at(-1);
+    assertExists(rubricMetric);
+    const rubric = String(rubricMetric.config?.rubric ?? "");
+    for (const amount of ["$15.21", "$99.71", "$33.24", "$33.23"]) {
+      assertEquals(
+        rubric.includes(amount),
+        true,
+        `the rubric should name the expected ${amount}`,
+      );
     }
-    for (
-      const invalid of [
-        "-15.21",
-        "-$15.21",
-        String.raw`-\$15.21`,
-        "115.21",
-        "$15.210",
-        "$15.21.0",
-      ]
-    ) {
-      assertEquals((await tipMetric.evaluate(createRecord(invalid))).pass, false);
+    assertEquals(
+      /exact/i.test(rubric),
+      true,
+      "the rubric should require exact amounts now that no regex enforces it",
+    );
+    for (const nearMiss of ["$33.2366", "$133.23"]) {
+      assertEquals(
+        rubric.includes(nearMiss),
+        true,
+        `the rubric should show ${nearMiss} as a failing near-miss`,
+      );
     }
   });
 
diff --git a/cli/templates/manifest.json b/cli/templates/manifest.json
index 7559281641..ba55bfdf5a 100644
--- a/cli/templates/manifest.json
+++ b/cli/templates/manifest.json
@@ -27,7 +27,7 @@
         "app/layout.tsx": "import \"../globals.css\";\nimport { Head } from \"veryfront/head\";\n\nexport default function RootLayout({\n  children,\n}: {\n  children: React.ReactNode;\n}): React.ReactNode {\n  return (\n    <>\n      <Head>\n        <title>Assistant</title>\n        <link rel=\"icon\" href=\"/favicon.svg\" type=\"image/svg+xml\" />\n      </Head>\n      {children}\n    </>\n  );\n}\n",
         "app/markdown-renderer.tsx": "\"use client\";\n\nimport ReactMarkdown from \"react-markdown\";\nimport remarkGfm from \"remark-gfm\";\nimport type { MarkdownRendererProps } from \"veryfront/markdown\";\n\n/**\n * Rich Markdown for assistant answers.\n *\n * `veryfront/markdown` presents plain source until a renderer is installed, so\n * this component supplies one. Swap in any renderer that accepts\n * `MarkdownRendererProps` to change how answers are parsed and rendered.\n */\nexport function MarkdownRenderer({ source }: MarkdownRendererProps): React.JSX.Element {\n  return <ReactMarkdown remarkPlugins={[remarkGfm]}>{source}</ReactMarkdown>;\n}\n",
         "app/page.tsx": "\"use client\";\n\nimport { Chat } from \"veryfront/chat\";\nimport { MarkdownRendererProvider } from \"veryfront/markdown\";\nimport { MarkdownRenderer } from \"./markdown-renderer.tsx\";\n\nexport default function ChatPage(): React.JSX.Element {\n  return (\n    <MarkdownRendererProvider renderer={MarkdownRenderer}>\n      <Chat agentId=\"assistant\" className=\"h-screen\" />\n    </MarkdownRendererProvider>\n  );\n}\n",
-        "evals/assistant.eval.ts": "import { datasets, evalAgent, judges, metrics } from \"veryfront/eval\";\n\nexport default evalAgent({\n  name: \"Assistant smoke test\",\n  target: \"agent:assistant\",\n  dataset: datasets.inline([\n    {\n      id: \"calculator\",\n      input:\n        \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n      reference: \"$99.71 total; two people pay $33.24 and one pays $33.23.\",\n    },\n  ]),\n  metrics: [\n    metrics.answer.regex({\n      pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$15\\.21(?!\\d|\\.\\d)`,\n    }).gate(),\n    metrics.answer.regex({\n      pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$99\\.71(?!\\d|\\.\\d)`,\n    }).gate(),\n    metrics.answer.regex({\n      pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$33\\.24(?!\\d|\\.\\d)`,\n    }).gate(),\n    metrics.answer.regex({\n      pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$33\\.23(?!\\d|\\.\\d)`,\n    }).gate(),\n    metrics.agent.calledTool(\"calculator\").gate(),\n    metrics.agent.noFailedTools().gate(),\n    metrics.judge.rubric({\n      rubric:\n        \"The answer must correctly state the $15.21 tip, $99.71 total, and a cent-exact split of two payments of $33.24 and one of $33.23. It should explain the result briefly.\",\n      judge: judges.llm.rubric(),\n    }).gate({ min: 0.8 }),\n  ],\n});\n",
+        "evals/assistant.eval.ts": "import { datasets, evalAgent, judges, metrics } from \"veryfront/eval\";\n\n// An eval runs your agent against fixed inputs and grades the results.\n// This one asks the assistant a single arithmetic question, then checks that it\n// used the calculator, that no tool errored, and that the answer is correct.\n//\n// Run it with: npm run eval -- assistant\nexport default evalAgent({\n  name: \"Assistant smoke test\",\n  target: \"agent:assistant\",\n\n  // The questions to ask. `reference` is the answer you expect; the judge below\n  // grades the agent's answer against it.\n  dataset: datasets.inline([\n    {\n      id: \"calculator\",\n      input:\n        \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n      reference:\n        \"The tip is $15.21 and the total is $99.71. Two people pay $33.24 and one pays $33.23.\",\n    },\n  ]),\n\n  // Each metric is a gate: if any one fails, the eval fails.\n  metrics: [\n    // The agent must do the arithmetic with the calculator tool, not in its head.\n    metrics.agent.calledTool(\"calculator\").gate(),\n\n    // No tool call may error.\n    metrics.agent.noFailedTools().gate(),\n\n    // A second model reads the answer and scores it from 0 to 1 against this\n    // rubric. It needs at least 0.8 to pass.\n    metrics.judge.rubric({\n      rubric: [\n        \"The answer must state a tip of $15.21 and a total of $99.71.\",\n        \"It must split the total into $33.24, $33.24, and $33.23.\",\n        \"Every amount must be exact to the cent: $33.2366 and $133.23 are wrong.\",\n        \"The explanation must be brief.\",\n      ].join(\" \"),\n      judge: judges.llm.rubric(),\n    }).gate({ min: 0.8 }),\n  ],\n});\n",
         "globals.css": "@import \"tailwindcss\";\n",
         "public/favicon.svg": "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 64 64\">\n  <rect width=\"64\" height=\"64\" fill=\"#fff\"/>\n  <circle cx=\"32\" cy=\"32\" r=\"20\" fill=\"#000\"/>\n</svg>\n",
         "README.md": "# AI Agent\n\nA small, customizable agent with a streaming chat UI and tool support.\n\n## What's included\n\n- Single assistant agent with streaming chat UI\n- Example calculator tool\n- Smoke eval for the agent and calculator\n- App-mode `Chat` component for real-time responses\n\n## Structure\n\n```\nagents/assistant.ts    Agent definition\ntools/calculator.ts    Example tool\nevals/assistant.eval.ts Agent smoke eval\napp/\n  api/ag-ui/route.ts    AG-UI endpoint\n  page.tsx             Chat interface\n```\n\n## Customize\n\n- Edit `agents/assistant.ts` to change the agent's identity, instructions, and suggestions.\n- Add or replace files in `tools/` to give the agent new capabilities.\n- Update `evals/assistant.eval.ts`, then run `npm run eval -- assistant`.\n- Edit `app/page.tsx` when you need to customize the chat UI.\n",