From 76837776698325a441cd72895b99062f72b5bc3a Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Wed, 29 Jul 2026 08:34:08 +0200 Subject: [PATCH 1/3] Make starter quality gates reject incorrect money splits The first public rubric-judge smoke run exposed two gaps: a substring assertion accepted 33.2366 as 33.23, and the starter calculator could not perform the rounding operation selected by the agent. Require bounded positive dollar amounts, teach cent-exact allocation, and support safe decimal rounding before publishing the next patch. Constraint: The starter eval must remain fast and understandable while combining deterministic and model-based grading. Rejected: Remove the no-failed-tools gate | would hide unsupported calculator calls. Rejected: Rely only on the LLM judge | would make exact monetary correctness nondeterministic. Confidence: high Scope-risk: narrow Directive: Keep deterministic currency gates alongside the rubric judge; do not replace them with substring checks. Tested: Template test; manifests; fmt; lint; typecheck; live 3-repetition eval with 3/3 passing all gates. Not-tested: Full suite has two unrelated baseline failures in docs coverage and a missing generated npm artifact. --- .../files/ai-agent/agents/assistant.ts | 2 +- .../files/ai-agent/evals/assistant.eval.ts | 8 +-- .../files/ai-agent/tools/calculator.ts | 7 ++- cli/templates/index.test.ts | 49 +++++++++++++++++-- cli/templates/manifest.json | 6 +-- deno.json | 2 +- src/utils/version-constant.ts | 2 +- 7 files changed, 61 insertions(+), 15 deletions(-) diff --git a/cli/templates/files/ai-agent/agents/assistant.ts b/cli/templates/files/ai-agent/agents/assistant.ts index 6765958f3a..2298f3f470 100644 --- a/cli/templates/files/ai-agent/agents/assistant.ts +++ b/cli/templates/files/ai-agent/agents/assistant.ts @@ -5,7 +5,7 @@ export default agent({ name: "Assistant", description: "Turn a rough idea into a clear next move.", system: - "Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. Use other tools when they improve accuracy, and state assumptions that affect the result.", + "Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. For currency splits, make rounded shares add exactly to the total and explain any remainder. Use other tools when they improve accuracy, and state assumptions that affect the result.", tools: true, maxSteps: 10, suggestions: { diff --git a/cli/templates/files/ai-agent/evals/assistant.eval.ts b/cli/templates/files/ai-agent/evals/assistant.eval.ts index e3afbdd826..5b13ffe80c 100644 --- a/cli/templates/files/ai-agent/evals/assistant.eval.ts +++ b/cli/templates/files/ai-agent/evals/assistant.eval.ts @@ -12,10 +12,10 @@ export default evalAgent({ }, ]), metrics: [ - metrics.answer.contains({ text: "15.21" }).gate(), - metrics.answer.contains({ text: "99.71" }).gate(), - metrics.answer.contains({ text: "33.24" }).gate(), - metrics.answer.contains({ text: "33.23" }).gate(), + metrics.answer.regex({ pattern: String.raw`(? v.object({ - operation: v.enum(["add", "subtract", "multiply", "divide"]), + operation: v.enum(["add", "subtract", "multiply", "divide", "round"]), a: v.number(), b: v.number(), }))(), execute: ({ operation, a, b }) => { + const precision = Math.min(100, Math.max(0, Math.trunc(b))); + if (operation === "divide" && b === 0) { throw new Error("Cannot divide by zero"); } @@ -17,6 +19,7 @@ export default tool({ if (operation === "add") return { result: a + b }; if (operation === "subtract") return { result: a - b }; if (operation === "multiply") return { result: a * b }; + if (operation === "round") return { result: Number(a.toFixed(precision)) }; return { result: a / b }; }, }); diff --git a/cli/templates/index.test.ts b/cli/templates/index.test.ts index 803b4c6889..039b77a89f 100644 --- a/cli/templates/index.test.ts +++ b/cli/templates/index.test.ts @@ -91,6 +91,20 @@ describe("cli/templates", () => { assertEquals(calculator.includes("execute: async"), false); assertEquals(calculator.includes("execute: ({ operation, a, b }) =>"), true); + assertEquals( + calculator.includes('v.enum(["add", "subtract", "multiply", "divide", "round"])'), + true, + ); + assertEquals( + calculator.includes("const precision = Math.min(100, Math.max(0, Math.trunc(b)));"), + true, + ); + assertEquals( + calculator.includes( + 'if (operation === "round") return { result: Number(a.toFixed(precision)) };', + ), + true, + ); }); it("keeps the ai-agent starter slim, actionable, and viewport-bound", async () => { @@ -112,6 +126,12 @@ describe("cli/templates", () => { agent.includes("Use the calculator tool for arithmetic instead of calculating mentally."), true, ); + assertEquals( + agent.includes( + "For currency splits, make rounded shares add exactly to the total and explain any remainder.", + ), + true, + ); assertEquals( agent.includes( 'prompt: "Turn this rough idea into a focused plan with the first three steps: "', @@ -133,11 +153,34 @@ describe("cli/templates", () => { ), true, ); - assertEquals(assistantEval.includes('metrics.answer.contains({ text: "15.21" }).gate()'), true); + assertEquals( + assistantEval.includes( + "metrics.answer.regex({ pattern: String.raw`(?\n \n Assistant\n \n \n {children}\n \n );\n}\n", "app/page.tsx": "\"use client\";\n\nimport { Chat } from \"veryfront/chat\";\n\nexport default function ChatPage(): React.JSX.Element {\n return ;\n}\n", - "evals/assistant.eval.ts": "import { datasets, evalAgent, judges, metrics } from \"veryfront/eval\";\n\nexport default evalAgent({\n name: \"Assistant smoke test\",\n target: \"agent:assistant\",\n dataset: datasets.inline([\n {\n id: \"calculator\",\n input:\n \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n reference: \"$99.71 total; two people pay $33.24 and one pays $33.23.\",\n },\n ]),\n metrics: [\n metrics.answer.contains({ text: \"15.21\" }).gate(),\n metrics.answer.contains({ text: \"99.71\" }).gate(),\n metrics.answer.contains({ text: \"33.24\" }).gate(),\n metrics.answer.contains({ text: \"33.23\" }).gate(),\n metrics.agent.calledTool(\"calculator\").gate(),\n metrics.agent.noFailedTools().gate(),\n metrics.judge.rubric({\n rubric:\n \"The answer must correctly state the $15.21 tip, $99.71 total, and a cent-exact split of two payments of $33.24 and one of $33.23. It should explain the result briefly.\",\n judge: judges.llm.rubric(),\n }).gate({ min: 0.8 }),\n ],\n});\n", + "evals/assistant.eval.ts": "import { datasets, evalAgent, judges, metrics } from \"veryfront/eval\";\n\nexport default evalAgent({\n name: \"Assistant smoke test\",\n target: \"agent:assistant\",\n dataset: datasets.inline([\n {\n id: \"calculator\",\n input:\n \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n reference: \"$99.71 total; two people pay $33.24 and one pays $33.23.\",\n },\n ]),\n metrics: [\n metrics.answer.regex({ pattern: String.raw`(?\n \n \n\n", "README.md": "# AI Agent\n\nA small, customizable agent with a streaming chat UI and tool support.\n\n## What's included\n\n- Single assistant agent with streaming chat UI\n- Example calculator tool\n- Smoke eval for the agent and calculator\n- App-mode `Chat` component for real-time responses\n\n## Structure\n\n```\nagents/assistant.ts Agent definition\ntools/calculator.ts Example tool\nevals/assistant.eval.ts Agent smoke eval\napp/\n api/ag-ui/route.ts AG-UI endpoint\n page.tsx Chat interface\n```\n\n## Customize\n\n- Edit `agents/assistant.ts` to change the agent's identity, instructions, and suggestions.\n- Add or replace files in `tools/` to give the agent new capabilities.\n- Update `evals/assistant.eval.ts`, then run `npm run eval -- assistant`.\n- Edit `app/page.tsx` when you need to customize the chat UI.\n", - "tools/calculator.ts": "import { tool } from \"veryfront/tool\";\nimport { defineSchema } from \"veryfront/schemas\";\n\nexport default tool({\n id: \"calculator\",\n description: \"Perform basic arithmetic operations\",\n inputSchema: defineSchema((v) => v.object({\n operation: v.enum([\"add\", \"subtract\", \"multiply\", \"divide\"]),\n a: v.number(),\n b: v.number(),\n }))(),\n execute: ({ operation, a, b }) => {\n if (operation === \"divide\" && b === 0) {\n throw new Error(\"Cannot divide by zero\");\n }\n\n if (operation === \"add\") return { result: a + b };\n if (operation === \"subtract\") return { result: a - b };\n if (operation === \"multiply\") return { result: a * b };\n return { result: a / b };\n },\n});\n", + "tools/calculator.ts": "import { tool } from \"veryfront/tool\";\nimport { defineSchema } from \"veryfront/schemas\";\n\nexport default tool({\n id: \"calculator\",\n description: \"Perform arithmetic. For round, a is the value and b is the decimal places.\",\n inputSchema: defineSchema((v) => v.object({\n operation: v.enum([\"add\", \"subtract\", \"multiply\", \"divide\", \"round\"]),\n a: v.number(),\n b: v.number(),\n }))(),\n execute: ({ operation, a, b }) => {\n const precision = Math.min(100, Math.max(0, Math.trunc(b)));\n\n if (operation === \"divide\" && b === 0) {\n throw new Error(\"Cannot divide by zero\");\n }\n\n if (operation === \"add\") return { result: a + b };\n if (operation === \"subtract\") return { result: a - b };\n if (operation === \"multiply\") return { result: a * b };\n if (operation === \"round\") return { result: Number(a.toFixed(precision)) };\n return { result: a / b };\n },\n});\n", "tsconfig.json": "{\n \"compilerOptions\": {\n \"target\": \"ES2022\",\n \"module\": \"ESNext\",\n \"moduleResolution\": \"bundler\",\n \"strict\": true,\n \"jsx\": \"react-jsx\",\n \"skipLibCheck\": true,\n \"esModuleInterop\": true,\n \"paths\": {\n \"@/*\": [\"./*\"]\n }\n },\n \"include\": [\"**/*.ts\", \"**/*.tsx\"],\n \"exclude\": [\"node_modules\"]\n}\n" } }, diff --git a/deno.json b/deno.json index 8f873989b7..4a8a482644 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "veryfront", - "version": "0.1.1173", + "version": "0.1.1174", "license": "Apache-2.0", "nodeModulesDir": "auto", "minimumDependencyAge": { diff --git a/src/utils/version-constant.ts b/src/utils/version-constant.ts index b9441c128f..5dbfce8574 100644 --- a/src/utils/version-constant.ts +++ b/src/utils/version-constant.ts @@ -1,4 +1,4 @@ // Keep in sync with deno.json version. // scripts/release.ts updates this constant during releases. /** Shared version value. */ -export const VERSION = "0.1.1173"; +export const VERSION = "0.1.1174"; From a4459c24a6c3b588657620b23615b412dd256e1a Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Wed, 29 Jul 2026 08:44:28 +0200 Subject: [PATCH 2/3] Keep generated money calculations cent-exact Binary floating-point representation can place half-cent values just below their intended decimal boundary. Apply a sign-aware, magnitude-scaled one-ULP offset before decimal formatting so the starter handles positive and negative currency values symmetrically. Constraint: Keep the starter dependency-free and preserve the existing 0-100 precision contract. Rejected: Raw toFixed rounding | produces 1.00 for 1.005 at two decimals. Confidence: high Scope-risk: narrow Directive: Keep the source template and generated manifest synchronized. Tested: Template suite including positive and negative half-cent execution; manifest check; full lint; full typecheck. --- .../files/ai-agent/tools/calculator.ts | 5 ++++- cli/templates/index.test.ts | 21 ++++++++++++++++++- cli/templates/manifest.json | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/cli/templates/files/ai-agent/tools/calculator.ts b/cli/templates/files/ai-agent/tools/calculator.ts index 7b7ede3954..8c1cc8f9b9 100644 --- a/cli/templates/files/ai-agent/tools/calculator.ts +++ b/cli/templates/files/ai-agent/tools/calculator.ts @@ -19,7 +19,10 @@ export default tool({ if (operation === "add") return { result: a + b }; if (operation === "subtract") return { result: a - b }; if (operation === "multiply") return { result: a * b }; - if (operation === "round") return { result: Number(a.toFixed(precision)) }; + if (operation === "round") { + const offset = Math.sign(a) * Number.EPSILON * Math.max(1, Math.abs(a)); + return { result: Number((a + offset).toFixed(precision)) }; + } return { result: a / b }; }, }); diff --git a/cli/templates/index.test.ts b/cli/templates/index.test.ts index 039b77a89f..dd3568c99f 100644 --- a/cli/templates/index.test.ts +++ b/cli/templates/index.test.ts @@ -101,10 +101,29 @@ describe("cli/templates", () => { ); assertEquals( calculator.includes( - 'if (operation === "round") return { result: Number(a.toFixed(precision)) };', + "const offset = Math.sign(a) * Number.EPSILON * Math.max(1, Math.abs(a));", ), true, ); + assertEquals( + calculator.includes("return { result: Number((a + offset).toFixed(precision)) };"), + true, + ); + }); + + it("rounds positive and negative half cents away from zero", async () => { + const { default: calculator } = await import( + "./files/ai-agent/tools/calculator.ts" + ); + + assertEquals( + await calculator.execute({ operation: "round", a: 1.005, b: 2 }), + { result: 1.01 }, + ); + assertEquals( + await calculator.execute({ operation: "round", a: -1.005, b: 2 }), + { result: -1.01 }, + ); }); it("keeps the ai-agent starter slim, actionable, and viewport-bound", async () => { diff --git a/cli/templates/manifest.json b/cli/templates/manifest.json index 78753dc97a..b0bf49ce1b 100644 --- a/cli/templates/manifest.json +++ b/cli/templates/manifest.json @@ -30,7 +30,7 @@ "globals.css": "@import \"tailwindcss\";\n", "public/favicon.svg": "\n \n \n\n", "README.md": "# AI Agent\n\nA small, customizable agent with a streaming chat UI and tool support.\n\n## What's included\n\n- Single assistant agent with streaming chat UI\n- Example calculator tool\n- Smoke eval for the agent and calculator\n- App-mode `Chat` component for real-time responses\n\n## Structure\n\n```\nagents/assistant.ts Agent definition\ntools/calculator.ts Example tool\nevals/assistant.eval.ts Agent smoke eval\napp/\n api/ag-ui/route.ts AG-UI endpoint\n page.tsx Chat interface\n```\n\n## Customize\n\n- Edit `agents/assistant.ts` to change the agent's identity, instructions, and suggestions.\n- Add or replace files in `tools/` to give the agent new capabilities.\n- Update `evals/assistant.eval.ts`, then run `npm run eval -- assistant`.\n- Edit `app/page.tsx` when you need to customize the chat UI.\n", - "tools/calculator.ts": "import { tool } from \"veryfront/tool\";\nimport { defineSchema } from \"veryfront/schemas\";\n\nexport default tool({\n id: \"calculator\",\n description: \"Perform arithmetic. For round, a is the value and b is the decimal places.\",\n inputSchema: defineSchema((v) => v.object({\n operation: v.enum([\"add\", \"subtract\", \"multiply\", \"divide\", \"round\"]),\n a: v.number(),\n b: v.number(),\n }))(),\n execute: ({ operation, a, b }) => {\n const precision = Math.min(100, Math.max(0, Math.trunc(b)));\n\n if (operation === \"divide\" && b === 0) {\n throw new Error(\"Cannot divide by zero\");\n }\n\n if (operation === \"add\") return { result: a + b };\n if (operation === \"subtract\") return { result: a - b };\n if (operation === \"multiply\") return { result: a * b };\n if (operation === \"round\") return { result: Number(a.toFixed(precision)) };\n return { result: a / b };\n },\n});\n", + "tools/calculator.ts": "import { tool } from \"veryfront/tool\";\nimport { defineSchema } from \"veryfront/schemas\";\n\nexport default tool({\n id: \"calculator\",\n description: \"Perform arithmetic. For round, a is the value and b is the decimal places.\",\n inputSchema: defineSchema((v) => v.object({\n operation: v.enum([\"add\", \"subtract\", \"multiply\", \"divide\", \"round\"]),\n a: v.number(),\n b: v.number(),\n }))(),\n execute: ({ operation, a, b }) => {\n const precision = Math.min(100, Math.max(0, Math.trunc(b)));\n\n if (operation === \"divide\" && b === 0) {\n throw new Error(\"Cannot divide by zero\");\n }\n\n if (operation === \"add\") return { result: a + b };\n if (operation === \"subtract\") return { result: a - b };\n if (operation === \"multiply\") return { result: a * b };\n if (operation === \"round\") {\n const offset = Math.sign(a) * Number.EPSILON * Math.max(1, Math.abs(a));\n return { result: Number((a + offset).toFixed(precision)) };\n }\n return { result: a / b };\n },\n});\n", "tsconfig.json": "{\n \"compilerOptions\": {\n \"target\": \"ES2022\",\n \"module\": \"ESNext\",\n \"moduleResolution\": \"bundler\",\n \"strict\": true,\n \"jsx\": \"react-jsx\",\n \"skipLibCheck\": true,\n \"esModuleInterop\": true,\n \"paths\": {\n \"@/*\": [\"./*\"]\n }\n },\n \"include\": [\"**/*.ts\", \"**/*.tsx\"],\n \"exclude\": [\"node_modules\"]\n}\n" } }, From 06bf8c52feb0a20e4f13b9c4837a467378813062 Mon Sep 17 00:00:00 2001 From: Koji Wakayama Date: Wed, 29 Jul 2026 08:47:34 +0200 Subject: [PATCH 3/3] Reject escaped negative amounts in starter evals Markdown-escaped dollar signs allowed the regex engine to restart at the dollar token and skip a preceding minus. Match the optional escape as part of the amount token and reject backslashes at the lookbehind boundary. Constraint: Continue accepting both plain and Markdown-escaped positive dollar amounts. Rejected: Disallow escaped dollar signs | valid model Markdown output uses them. Confidence: high Scope-risk: narrow Directive: Apply the same token boundary to every exact-money gate in this starter. Tested: Regression for escaped negative amount; template suite; manifest check; focused lint; full typecheck. --- .../files/ai-agent/evals/assistant.eval.ts | 16 ++++++++++++---- cli/templates/index.test.ts | 19 ++++++++++++++----- cli/templates/manifest.json | 2 +- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/cli/templates/files/ai-agent/evals/assistant.eval.ts b/cli/templates/files/ai-agent/evals/assistant.eval.ts index 5b13ffe80c..083fb1aee5 100644 --- a/cli/templates/files/ai-agent/evals/assistant.eval.ts +++ b/cli/templates/files/ai-agent/evals/assistant.eval.ts @@ -12,10 +12,18 @@ export default evalAgent({ }, ]), metrics: [ - metrics.answer.regex({ pattern: String.raw`(? { ); assertEquals( assistantEval.includes( - "metrics.answer.regex({ pattern: String.raw`(? { assertEquals(assistantEval.includes("metrics.agent.noFailedTools().gate()"), true); assertEquals( assistantEval.includes( - "metrics.answer.regex({ pattern: String.raw`(?\n \n Assistant\n \n \n {children}\n \n );\n}\n", "app/page.tsx": "\"use client\";\n\nimport { Chat } from \"veryfront/chat\";\n\nexport default function ChatPage(): React.JSX.Element {\n return ;\n}\n", - "evals/assistant.eval.ts": "import { datasets, evalAgent, judges, metrics } from \"veryfront/eval\";\n\nexport default evalAgent({\n name: \"Assistant smoke test\",\n target: \"agent:assistant\",\n dataset: datasets.inline([\n {\n id: \"calculator\",\n input:\n \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n reference: \"$99.71 total; two people pay $33.24 and one pays $33.23.\",\n },\n ]),\n metrics: [\n metrics.answer.regex({ pattern: String.raw`(?\n \n \n\n", "README.md": "# AI Agent\n\nA small, customizable agent with a streaming chat UI and tool support.\n\n## What's included\n\n- Single assistant agent with streaming chat UI\n- Example calculator tool\n- Smoke eval for the agent and calculator\n- App-mode `Chat` component for real-time responses\n\n## Structure\n\n```\nagents/assistant.ts Agent definition\ntools/calculator.ts Example tool\nevals/assistant.eval.ts Agent smoke eval\napp/\n api/ag-ui/route.ts AG-UI endpoint\n page.tsx Chat interface\n```\n\n## Customize\n\n- Edit `agents/assistant.ts` to change the agent's identity, instructions, and suggestions.\n- Add or replace files in `tools/` to give the agent new capabilities.\n- Update `evals/assistant.eval.ts`, then run `npm run eval -- assistant`.\n- Edit `app/page.tsx` when you need to customize the chat UI.\n",