Skip to content

chore(templates): make the starter eval readable #3426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kwakayama merged 1 commit into from
Aug 6, 2026
Merged

chore(templates): make the starter eval readable #3426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 24 additions & 15 deletions cli/templates/files/ai-agent/evals/assistant.eval.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,34 +1,43 @@
import { datasets, evalAgent, judges, metrics } from "veryfront/eval";

// An eval runs your agent against fixed inputs and grades the results.
// This one asks the assistant a single arithmetic question, then checks that it
// used the calculator, that no tool errored, and that the answer is correct.
//
// Run it with: npm run eval -- assistant
export default evalAgent({
name: "Assistant smoke test",
target: "agent:assistant",

// The questions to ask. `reference` is the answer you expect; the judge below
// grades the agent's answer against it.
dataset: datasets.inline([
{
id: "calculator",
input:
"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.",
reference: "$99.71 total; two people pay $33.24 and one pays $33.23.",
reference:
"The tip is $15.21 and the total is $99.71. Two people pay $33.24 and one pays $33.23.",
},
]),

// Each metric is a gate: if any one fails, the eval fails.
metrics: [
metrics.answer.regex({
pattern: String.raw`(?<![-\d.\\])\\?\$15\.21(?!\d|\.\d)`,
}).gate(),
metrics.answer.regex({
pattern: String.raw`(?<![-\d.\\])\\?\$99\.71(?!\d|\.\d)`,
}).gate(),
metrics.answer.regex({
pattern: String.raw`(?<![-\d.\\])\\?\$33\.24(?!\d|\.\d)`,
}).gate(),
metrics.answer.regex({
pattern: String.raw`(?<![-\d.\\])\\?\$33\.23(?!\d|\.\d)`,
}).gate(),
// The agent must do the arithmetic with the calculator tool, not in its head.
metrics.agent.calledTool("calculator").gate(),

// No tool call may error.
metrics.agent.noFailedTools().gate(),

// A second model reads the answer and scores it from 0 to 1 against this
// rubric. It needs at least 0.8 to pass.
metrics.judge.rubric({
rubric:
"The answer must correctly state the $15.21 tip, $99.71 total, and a cent-exact split of two payments of $33.24 and one of $33.23. It should explain the result briefly.",
rubric: [
"The answer must state a tip of $15.21 and a total of $99.71.",
"It must split the total into $33.24, $33.24, and $33.23.",
"Every amount must be exact to the cent: $33.2366 and $133.23 are wrong.",
"The explanation must be brief.",
].join(" "),
judge: judges.llm.rubric(),
}).gate({ min: 0.8 }),
],
Expand Down
92 changes: 46 additions & 46 deletions cli/templates/index.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {
} from "#veryfront/testing/assert.ts";
import { describe, it } from "#veryfront/testing/bdd.ts";
import { fromFileUrl } from "#veryfront/compat/path";
import type { EvalRecord } from "veryfront/eval";

import { getTemplate, getTemplateConfig, templateConfigs } from "./index.ts";
import { STARTER_TEMPLATE_NAMES, type TemplateName } from "./types.ts";
Expand Down Expand Up @@ -179,59 +178,60 @@ describe("cli/templates", () => {
]);
});

it("accepts sentence punctuation without accepting longer monetary values", async () => {
it("grades the starter's money answer with a rubric a reader can follow", async () => {
// The starter eval is the first eval most people ever read, so it has to be
// legible. It used to gate each amount with a hand-rolled lookaround regex
// (`(?<![-\d.\\])\\?\$33\.23(?!\d|\.\d)`) to reject near-misses like
// $33.2366 and $133.23. That was exact but unreadable, and copy-pasting it
// is the wrong lesson to teach. The rubric judge now carries the exactness
// requirement in prose instead.
const { default: assistantEval } = await import(
"./files/ai-agent/evals/assistant.eval.ts"
);
const moneyMetrics = assistantEval.metrics.slice(0, 4);
assertEquals(moneyMetrics.map((metric) => metric.name), [
"answer.regex",
"answer.regex",
"answer.regex",
"answer.regex",

assertEquals(assistantEval.metrics.map((metric) => metric.name), [
"agent.calledTool",
"agent.noFailedTools",
"judge.rubric",
]);
const createRecord = (text: string): EvalRecord => ({
id: "calculator:1",
evalId: "eval:assistant",
exampleId: "calculator",
repetition: 1,
input: "Calculate the tip and split.",
output: { text },
reference: "$99.71 total; two people pay $33.24 and one pays $33.23.",
metadata: {},
trace: { events: [], toolCalls: [] },
usage: {},
durationMs: 1,
completed: true,
});

const validResults = await Promise.all(
moneyMetrics.map((metric) =>
metric.evaluate(
createRecord(
"The tip is $15.21. The total is $99.71. Two people pay $33.24, and one pays $33.23.",
),
)
),
const source = await Deno.readTextFile(
new URL("./files/ai-agent/evals/assistant.eval.ts", import.meta.url),
);
assertEquals(
source.includes("metrics.answer.regex"),
false,
"the starter eval should not teach hand-rolled regex assertions",
);
assertEquals(
source.includes("String.raw"),
false,
"the starter eval should not need raw strings to express an assertion",
);
assertEquals(validResults.map((result) => result.pass), [true, true, true, true]);

const tipMetric = moneyMetrics[0];
assertExists(tipMetric);
for (const valid of ["$15.21.", String.raw`\$15.21`, "($15.21)", "**$15.21**"]) {
assertEquals((await tipMetric.evaluate(createRecord(valid))).pass, true);
// Dropping the regexes moved exactness onto the judge, so the rubric has to
// spell out both the amounts and that near-misses fail.
const rubricMetric = assistantEval.metrics.at(-1);
assertExists(rubricMetric);
const rubric = String(rubricMetric.config?.rubric ?? "");
for (const amount of ["$15.21", "$99.71", "$33.24", "$33.23"]) {
assertEquals(
rubric.includes(amount),
true,
`the rubric should name the expected ${amount}`,
);
}
for (
const invalid of [
"-15.21",
"-$15.21",
String.raw`-\$15.21`,
"115.21",
"$15.210",
"$15.21.0",
]
) {
assertEquals((await tipMetric.evaluate(createRecord(invalid))).pass, false);
assertEquals(
/exact/i.test(rubric),
true,
"the rubric should require exact amounts now that no regex enforces it",
);
for (const nearMiss of ["$33.2366", "$133.23"]) {
assertEquals(
rubric.includes(nearMiss),
true,
`the rubric should show ${nearMiss} as a failing near-miss`,
);
}
});

Expand Down
2 changes: 1 addition & 1 deletion cli/templates/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
"app/layout.tsx": "import \"../globals.css\";\nimport { Head } from \"veryfront/head\";\n\nexport default function RootLayout({\n children,\n}: {\n children: React.ReactNode;\n}): React.ReactNode {\n return (\n <>\n <Head>\n <title>Assistant</title>\n <link rel=\"icon\" href=\"/favicon.svg\" type=\"image/svg+xml\" />\n </Head>\n {children}\n </>\n );\n}\n",
"app/markdown-renderer.tsx": "\"use client\";\n\nimport ReactMarkdown from \"react-markdown\";\nimport remarkGfm from \"remark-gfm\";\nimport type { MarkdownRendererProps } from \"veryfront/markdown\";\n\n/**\n * Rich Markdown for assistant answers.\n *\n * `veryfront/markdown` presents plain source until a renderer is installed, so\n * this component supplies one. Swap in any renderer that accepts\n * `MarkdownRendererProps` to change how answers are parsed and rendered.\n */\nexport function MarkdownRenderer({ source }: MarkdownRendererProps): React.JSX.Element {\n return <ReactMarkdown remarkPlugins={[remarkGfm]}>{source}</ReactMarkdown>;\n}\n",
"app/page.tsx": "\"use client\";\n\nimport { Chat } from \"veryfront/chat\";\nimport { MarkdownRendererProvider } from \"veryfront/markdown\";\nimport { MarkdownRenderer } from \"./markdown-renderer.tsx\";\n\nexport default function ChatPage(): React.JSX.Element {\n return (\n <MarkdownRendererProvider renderer={MarkdownRenderer}>\n <Chat agentId=\"assistant\" className=\"h-screen\" />\n </MarkdownRendererProvider>\n );\n}\n",
"evals/assistant.eval.ts": "import { datasets, evalAgent, judges, metrics } from \"veryfront/eval\";\n\nexport default evalAgent({\n name: \"Assistant smoke test\",\n target: \"agent:assistant\",\n dataset: datasets.inline([\n {\n id: \"calculator\",\n input:\n \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n reference: \"$99.71 total; two people pay $33.24 and one pays $33.23.\",\n },\n ]),\n metrics: [\n metrics.answer.regex({\n pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$15\\.21(?!\\d|\\.\\d)`,\n }).gate(),\n metrics.answer.regex({\n pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$99\\.71(?!\\d|\\.\\d)`,\n }).gate(),\n metrics.answer.regex({\n pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$33\\.24(?!\\d|\\.\\d)`,\n }).gate(),\n metrics.answer.regex({\n pattern: String.raw`(?<![-\\d.\\\\])\\\\?\\$33\\.23(?!\\d|\\.\\d)`,\n }).gate(),\n metrics.agent.calledTool(\"calculator\").gate(),\n metrics.agent.noFailedTools().gate(),\n metrics.judge.rubric({\n rubric:\n \"The answer must correctly state the $15.21 tip, $99.71 total, and a cent-exact split of two payments of $33.24 and one of $33.23. It should explain the result briefly.\",\n judge: judges.llm.rubric(),\n }).gate({ min: 0.8 }),\n ],\n});\n",
"evals/assistant.eval.ts": "import { datasets, evalAgent, judges, metrics } from \"veryfront/eval\";\n\n// An eval runs your agent against fixed inputs and grades the results.\n// This one asks the assistant a single arithmetic question, then checks that it\n// used the calculator, that no tool errored, and that the answer is correct.\n//\n// Run it with: npm run eval -- assistant\nexport default evalAgent({\n name: \"Assistant smoke test\",\n target: \"agent:assistant\",\n\n // The questions to ask. `reference` is the answer you expect; the judge below\n // grades the agent's answer against it.\n dataset: datasets.inline([\n {\n id: \"calculator\",\n input:\n \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n reference:\n \"The tip is $15.21 and the total is $99.71. Two people pay $33.24 and one pays $33.23.\",\n },\n ]),\n\n // Each metric is a gate: if any one fails, the eval fails.\n metrics: [\n // The agent must do the arithmetic with the calculator tool, not in its head.\n metrics.agent.calledTool(\"calculator\").gate(),\n\n // No tool call may error.\n metrics.agent.noFailedTools().gate(),\n\n // A second model reads the answer and scores it from 0 to 1 against this\n // rubric. It needs at least 0.8 to pass.\n metrics.judge.rubric({\n rubric: [\n \"The answer must state a tip of $15.21 and a total of $99.71.\",\n \"It must split the total into $33.24, $33.24, and $33.23.\",\n \"Every amount must be exact to the cent: $33.2366 and $133.23 are wrong.\",\n \"The explanation must be brief.\",\n ].join(\" \"),\n judge: judges.llm.rubric(),\n }).gate({ min: 0.8 }),\n ],\n});\n",
"globals.css": "@import \"tailwindcss\";\n",
"public/favicon.svg": "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 64 64\">\n <rect width=\"64\" height=\"64\" fill=\"#fff\"/>\n <circle cx=\"32\" cy=\"32\" r=\"20\" fill=\"#000\"/>\n</svg>\n",
"README.md": "# AI Agent\n\nA small, customizable agent with a streaming chat UI and tool support.\n\n## What's included\n\n- Single assistant agent with streaming chat UI\n- Example calculator tool\n- Smoke eval for the agent and calculator\n- App-mode `Chat` component for real-time responses\n\n## Structure\n\n```\nagents/assistant.ts Agent definition\ntools/calculator.ts Example tool\nevals/assistant.eval.ts Agent smoke eval\napp/\n api/ag-ui/route.ts AG-UI endpoint\n page.tsx Chat interface\n```\n\n## Customize\n\n- Edit `agents/assistant.ts` to change the agent's identity, instructions, and suggestions.\n- Add or replace files in `tools/` to give the agent new capabilities.\n- Update `evals/assistant.eval.ts`, then run `npm run eval -- assistant`.\n- Edit `app/page.tsx` when you need to customize the chat UI.\n",
Expand Down