fix(templates): make the ai-agent starter pass its own eval

cli/templates/files/ai-agent/agents/assistant.ts

@@ -5,8 +5,8 @@ export default agent({
   name: "Assistant",
   description: "Turn a rough idea into a clear next move.",
   system:
-    "Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. Plan the calculation before calling the calculator, use the fewest calls needed, and answer immediately after you have the result. For currency splits, make rounded shares add exactly to the total and explain any remainder. Use other tools when they improve accuracy, and state assumptions that affect the result.",
-  tools: true,
+    "Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. Plan the calculation before calling the calculator, use the fewest calls needed, and answer immediately after you have the result. For currency splits, make rounded shares add exactly to the total and explain any remainder. Write the numbers you get back in plain text, using x and / for operators, never in LaTeX or MathJax. Use other tools when they improve accuracy, and state assumptions that affect the result.",
+  tools: { calculator: true },
   maxSteps: 20,
   suggestions: [
     {

cli/templates/index.test.ts

@@ -6,6 +6,7 @@ import {
   assertThrows,
 } from "#veryfront/testing/assert.ts";
 import { describe, it } from "#veryfront/testing/bdd.ts";
+import { fromFileUrl } from "#veryfront/compat/path";
 import type { EvalRecord } from "veryfront/eval";
 
 import { getTemplate, getTemplateConfig, templateConfigs } from "./index.ts";
@@ -234,6 +235,78 @@ describe("cli/templates", () => {
     }
   });
 
+  it("offers every tool its eval gates on", async () => {
+    // `tools: true` does not expose project tools to the model. It authorizes
+    // the scoped catalog behind `tool_search`, so the model has to go looking
+    // first, and for easy work it simply will not. The ai-agent starter shipped
+    // that way: its prompt said to use the calculator and its eval gated on
+    // `calledTool("calculator")`, but the tool was never offered, so the demo
+    // failed its own eval while answering correctly.
+    const templatesDir = fromFileUrl(new URL("./files/", import.meta.url));
+    const offences: string[] = [];
+
+    for await (const entry of Deno.readDir(templatesDir)) {
+      if (!entry.isDirectory) continue;
+      const evalsDir = `${templatesDir}${entry.name}/evals`;
+
+      const gated = new Set<string>();
+      try {
+        for await (const file of Deno.readDir(evalsDir)) {
+          if (!file.name.endsWith(".ts")) continue;
+          const source = await Deno.readTextFile(`${evalsDir}/${file.name}`);
+          for (const match of source.matchAll(/calledTool\(\s*"([^"]+)"/g)) {
+            if (match[1]) gated.add(match[1]);
+          }
+        }
+      } catch {
+        continue;
+      }
+      if (gated.size === 0) continue;
+
+      let agents = "";
+      try {
+        for await (const file of Deno.readDir(`${templatesDir}${entry.name}/agents`)) {
+          if (!file.name.endsWith(".ts")) continue;
+          agents += await Deno.readTextFile(`${templatesDir}${entry.name}/agents/${file.name}`);
+        }
+      } catch {
+        // No agents directory: the gate cannot be satisfied at all.
+      }
+
+      for (const tool of gated) {
+        if (!new RegExp(`\\b${tool}\\s*:`).test(agents)) {
+          offences.push(`${entry.name} gates on ${tool}`);
+        }
+      }
+    }
+
+    assertEquals(
+      offences.toSorted(),
+      [],
+      "an eval gates on a tool the agent never names. `tools: true` is not " +
+        "enough: list the tool explicitly, as `tools: { name: true }`.",
+    );
+  });
+
+  it("keeps the eval's expected answers out of the system prompt", async () => {
+    // A worked example in the prompt lets the model copy the answer instead of
+    // computing it, which passes the money gates and the rubric while the
+    // calculator is never called. The gates then measure the prompt, not the
+    // agent.
+    const agent = await Deno.readTextFile(
+      new URL("./files/ai-agent/agents/assistant.ts", import.meta.url),
+    );
+    const system = /system:\s*\n?\s*"([\s\S]*?)",\n/.exec(agent)?.[1] ?? "";
+    assertEquals(system.length > 0, true, "could not read the system prompt");
+
+    const leaked = ["15.21", "99.71", "33.24", "33.23"].filter((amount) => system.includes(amount));
+    assertEquals(
+      leaked,
+      [],
+      "the system prompt states amounts the eval asserts the agent should derive",
+    );
+  });
+
   it("keeps the ai-agent starter slim, actionable, and viewport-bound", async () => {
     const agent = await Deno.readTextFile(
       new URL("./files/ai-agent/agents/assistant.ts", import.meta.url),

cli/templates/manifest.json

@@ -22,7 +22,7 @@
     },
     "ai-agent": {
       "files": {
-        "agents/assistant.ts": "import { agent } from \"veryfront/agent\";\n\nexport default agent({\n  id: \"assistant\",\n  name: \"Assistant\",\n  description: \"Turn a rough idea into a clear next move.\",\n  system:\n    \"Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. Plan the calculation before calling the calculator, use the fewest calls needed, and answer immediately after you have the result. For currency splits, make rounded shares add exactly to the total and explain any remainder. Use other tools when they improve accuracy, and state assumptions that affect the result.\",\n  tools: true,\n  maxSteps: 20,\n  suggestions: [\n    {\n      type: \"prompt\",\n      title: \"Shape an idea\",\n      prompt: \"Turn this rough idea into a focused plan with the first three steps: \",\n    },\n    {\n      type: \"prompt\",\n      title: \"Run the numbers\",\n      prompt:\n        \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n    },\n  ],\n});\n",
+        "agents/assistant.ts": "import { agent } from \"veryfront/agent\";\n\nexport default agent({\n  id: \"assistant\",\n  name: \"Assistant\",\n  description: \"Turn a rough idea into a clear next move.\",\n  system:\n    \"Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. Plan the calculation before calling the calculator, use the fewest calls needed, and answer immediately after you have the result. For currency splits, make rounded shares add exactly to the total and explain any remainder. Write the numbers you get back in plain text, using x and / for operators, never in LaTeX or MathJax. Use other tools when they improve accuracy, and state assumptions that affect the result.\",\n  tools: { calculator: true },\n  maxSteps: 20,\n  suggestions: [\n    {\n      type: \"prompt\",\n      title: \"Shape an idea\",\n      prompt: \"Turn this rough idea into a focused plan with the first three steps: \",\n    },\n    {\n      type: \"prompt\",\n      title: \"Run the numbers\",\n      prompt:\n        \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n    },\n  ],\n});\n",
         "app/api/ag-ui/route.ts": "import { createAgUiHandler } from \"veryfront/agent\";\n\nexport const POST = createAgUiHandler(\"assistant\");\n",
         "app/layout.tsx": "import \"../globals.css\";\nimport { Head } from \"veryfront/head\";\n\nexport default function RootLayout({\n  children,\n}: {\n  children: React.ReactNode;\n}): React.ReactNode {\n  return (\n    <>\n      <Head>\n        <title>Assistant</title>\n        <link rel=\"icon\" href=\"/favicon.svg\" type=\"image/svg+xml\" />\n      </Head>\n      {children}\n    </>\n  );\n}\n",
         "app/markdown-renderer.tsx": "\"use client\";\n\nimport ReactMarkdown from \"react-markdown\";\nimport remarkGfm from \"remark-gfm\";\nimport type { MarkdownRendererProps } from \"veryfront/markdown\";\n\n/**\n * Rich Markdown for assistant answers.\n *\n * `veryfront/markdown` presents plain source until a renderer is installed, so\n * this component supplies one. Swap in any renderer that accepts\n * `MarkdownRendererProps` to change how answers are parsed and rendered.\n */\nexport function MarkdownRenderer({ source }: MarkdownRendererProps): React.JSX.Element {\n  return <ReactMarkdown remarkPlugins={[remarkGfm]}>{source}</ReactMarkdown>;\n}\n",