release: v0.1.1212 — ai-agent starter passes its own eval out of the box

cli/templates/files/ai-agent/agents/assistant.ts

@@ -5,7 +5,7 @@ export default agent({
   name: "Assistant",
   description: "Turn a rough idea into a clear next move.",
   system:
-    "Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. Plan the calculation before calling the calculator, use the fewest calls needed, and answer immediately after you have the result. For currency splits, make rounded shares add exactly to the total and explain any remainder. Write the numbers you get back in plain text, using x and / for operators, never in LaTeX or MathJax. Use other tools when they improve accuracy, and state assumptions that affect the result.",
+    "Be direct and practical. Use the calculator tool for arithmetic instead of calculating mentally, and answer as soon as you have the result. For currency splits use the calculator's split operation, then state every share it returns. Write numbers in plain text, using x and / for operators, never in LaTeX or MathJax.",
   tools: { calculator: true },
   maxSteps: 20,
   suggestions: [

cli/templates/files/ai-agent/tools/calculator.ts

@@ -3,26 +3,38 @@ import { defineSchema } from "veryfront/schemas";
 
 export default tool({
   id: "calculator",
-  description: "Perform arithmetic. For round, a is the value and b is the decimal places.",
-  inputSchema: defineSchema((v) => v.object({
-    operation: v.enum(["add", "subtract", "multiply", "divide", "round"]),
-    a: v.number(),
-    b: v.number(),
-  }))(),
+  description:
+    "Perform one arithmetic operation on two numbers. Use split to divide a money amount a into b shares that add up to it exactly.",
+  inputSchema: defineSchema((v) =>
+    v.object({
+      operation: v.enum(["add", "subtract", "multiply", "divide", "split"]),
+      a: v.number(),
+      b: v.number(),
+    })
+  )(),
   execute: ({ operation, a, b }) => {
-    const precision = Math.min(100, Math.max(0, Math.trunc(b)));
-
-    if (operation === "divide" && b === 0) {
+    if ((operation === "divide" || operation === "split") && b === 0) {
       throw new Error("Cannot divide by zero");
     }
 
+    if (operation === "split") {
+      const parts = Math.max(1, Math.trunc(Math.abs(b)));
+      if (parts > 1000) throw new Error("Cannot split into more than 1000 shares");
+
+      const cents = Math.round(a * 100);
+      const base = Math.trunc(cents / parts);
+      const remainder = Math.abs(cents - base * parts);
+      return {
+        result: Array.from(
+          { length: parts },
+          (_, index) => (base + (index < remainder ? Math.sign(cents) : 0)) / 100,
+        ),
+      };
+    }
+
     if (operation === "add") return { result: a + b };
     if (operation === "subtract") return { result: a - b };
     if (operation === "multiply") return { result: a * b };
-    if (operation === "round") {
-      const offset = Math.sign(a) * Number.EPSILON * Math.max(1, Math.abs(a));
-      return { result: Number((a + offset).toFixed(precision)) };
-    }
     return { result: a / b };
   },
 });

cli/templates/index.test.ts

@@ -109,37 +109,56 @@ describe("cli/templates", () => {
     assertEquals(calculator.includes("execute: async"), false);
     assertEquals(calculator.includes("execute: ({ operation, a, b }) =>"), true);
     assertEquals(
-      calculator.includes('v.enum(["add", "subtract", "multiply", "divide", "round"])'),
+      calculator.includes('v.enum(["add", "subtract", "multiply", "divide", "split"])'),
       true,
     );
-    assertEquals(
-      calculator.includes("const precision = Math.min(100, Math.max(0, Math.trunc(b)));"),
-      true,
+  });
+
+  it("keeps one meaning for every calculator argument", async () => {
+    const calculator = await Deno.readTextFile(
+      new URL("./files/ai-agent/tools/calculator.ts", import.meta.url),
     );
-    assertEquals(
-      calculator.includes(
-        "const offset = Math.sign(a) * Number.EPSILON * Math.max(1, Math.abs(a));",
-      ),
-      true,
+
+    assertEquals(calculator.includes('"round"'), false);
+    assertEquals(calculator.includes("decimals"), false);
+  });
+
+  it("splits money into shares that add up to the total exactly", async () => {
+    const { default: calculator } = await import(
+      "./files/ai-agent/tools/calculator.ts"
     );
+
     assertEquals(
-      calculator.includes("return { result: Number((a + offset).toFixed(precision)) };"),
-      true,
+      await calculator.execute({ operation: "split", a: 99.71, b: 3 }),
+      { result: [33.24, 33.24, 33.23] },
     );
+
+    for (const [total, ways] of [[99.71, 3], [0.01, 3], [10, 4], [-99.71, 3]] as const) {
+      const { result } = await calculator.execute({ operation: "split", a: total, b: ways });
+      if (!Array.isArray(result)) throw new Error("split should return one share per part");
+      assertEquals(result.length, ways);
+      assertEquals(
+        Math.round(result.reduce((sum, share) => sum + share, 0) * 100),
+        Math.round(total * 100),
+        `shares for ${total} split ${ways} ways should add back to the total`,
+      );
+    }
   });
 
-  it("rounds positive and negative half cents away from zero", async () => {
+  it("refuses a split count it cannot allocate", async () => {
     const { default: calculator } = await import(
       "./files/ai-agent/tools/calculator.ts"
     );
 
-    assertEquals(
-      await calculator.execute({ operation: "round", a: 1.005, b: 2 }),
-      { result: 1.01 },
+    await assertRejects(
+      () => calculator.execute({ operation: "split", a: 10, b: 2 ** 32 }),
+      Error,
+      "Cannot split into more than 1000 shares",
     );
-    assertEquals(
-      await calculator.execute({ operation: "round", a: -1.005, b: 2 }),
-      { result: -1.01 },
+    await assertRejects(
+      () => calculator.execute({ operation: "split", a: 10, b: 0 }),
+      Error,
+      "Cannot divide by zero",
     );
   });
 
@@ -152,7 +171,7 @@ describe("cli/templates", () => {
     assertEquals(
       typeof assistant.config.system === "string" &&
         assistant.config.system.includes(
-          "Plan the calculation before calling the calculator, use the fewest calls needed, and answer immediately after you have the result.",
+          "Use the calculator tool for arithmetic instead of calculating mentally, and answer as soon as you have the result.",
         ),
       true,
     );
@@ -208,13 +227,6 @@ describe("cli/templates", () => {
       "the starter eval should not need raw strings to express an assertion",
     );
 
-    // The rubric names the amounts it grades, and nothing more. Two further
-    // clauses used to ride along: one rejecting near-misses like $33.2366 and
-    // $133.23, one demanding a brief explanation. Both failed the 0.8 gate on a
-    // fresh scaffold, because the assistant routinely shows the repeating
-    // division before rounding and writes at length about the remainder. A
-    // starter eval that fails on the first run teaches nothing about the user's
-    // own setup, so the rubric asks only for the arithmetic.
     const rubricMetric = assistantEval.metrics.at(-1);
     assertExists(rubricMetric);
     const rubric = String(rubricMetric.config?.rubric ?? "");
@@ -322,12 +334,12 @@ describe("cli/templates", () => {
     assertEquals(agent.includes('name: "Assistant"'), true);
     assertEquals(agent.includes('description: "Turn a rough idea into a clear next move."'), true);
     assertEquals(
-      agent.includes("Use the calculator tool for arithmetic instead of calculating mentally."),
+      agent.includes("Use the calculator tool for arithmetic instead of calculating mentally"),
       true,
     );
     assertEquals(
       agent.includes(
-        "For currency splits, make rounded shares add exactly to the total and explain any remainder.",
+        "For currency splits use the calculator's split operation, then state every share it returns.",
       ),
       true,
     );

cli/templates/manifest.json

@@ -22,7 +22,7 @@
     },
     "ai-agent": {
       "files": {
-        "agents/assistant.ts": "import { agent } from \"veryfront/agent\";\n\nexport default agent({\n  id: \"assistant\",\n  name: \"Assistant\",\n  description: \"Turn a rough idea into a clear next move.\",\n  system:\n    \"Be direct and practical. Structure complex answers clearly. Use the calculator tool for arithmetic instead of calculating mentally. Plan the calculation before calling the calculator, use the fewest calls needed, and answer immediately after you have the result. For currency splits, make rounded shares add exactly to the total and explain any remainder. Write the numbers you get back in plain text, using x and / for operators, never in LaTeX or MathJax. Use other tools when they improve accuracy, and state assumptions that affect the result.\",\n  tools: { calculator: true },\n  maxSteps: 20,\n  suggestions: [\n    {\n      type: \"prompt\",\n      title: \"Shape an idea\",\n      prompt: \"Turn this rough idea into a focused plan with the first three steps: \",\n    },\n    {\n      type: \"prompt\",\n      title: \"Run the numbers\",\n      prompt:\n        \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n    },\n  ],\n});\n",
+        "agents/assistant.ts": "import { agent } from \"veryfront/agent\";\n\nexport default agent({\n  id: \"assistant\",\n  name: \"Assistant\",\n  description: \"Turn a rough idea into a clear next move.\",\n  system:\n    \"Be direct and practical. Use the calculator tool for arithmetic instead of calculating mentally, and answer as soon as you have the result. For currency splits use the calculator's split operation, then state every share it returns. Write numbers in plain text, using x and / for operators, never in LaTeX or MathJax.\",\n  tools: { calculator: true },\n  maxSteps: 20,\n  suggestions: [\n    {\n      type: \"prompt\",\n      title: \"Shape an idea\",\n      prompt: \"Turn this rough idea into a focused plan with the first three steps: \",\n    },\n    {\n      type: \"prompt\",\n      title: \"Run the numbers\",\n      prompt:\n        \"Calculate an 18% tip on $84.50, split the total among three people, and explain the result briefly.\",\n    },\n  ],\n});\n",
         "app/api/ag-ui/route.ts": "import { createAgUiHandler } from \"veryfront/agent\";\n\nexport const POST = createAgUiHandler(\"assistant\");\n",
         "app/layout.tsx": "import \"../globals.css\";\nimport { Head } from \"veryfront/head\";\n\nexport default function RootLayout({\n  children,\n}: {\n  children: React.ReactNode;\n}): React.ReactNode {\n  return (\n    <>\n      <Head>\n        <title>Assistant</title>\n        <link rel=\"icon\" href=\"/favicon.svg\" type=\"image/svg+xml\" />\n      </Head>\n      {children}\n    </>\n  );\n}\n",
         "app/markdown-renderer.tsx": "\"use client\";\n\nimport ReactMarkdown from \"react-markdown@9.0.3\";\nimport remarkGfm from \"remark-gfm@4.0.1\";\nimport type { MarkdownRendererProps } from \"veryfront/markdown\";\n\n/**\n * Rich Markdown for assistant answers.\n *\n * `veryfront/markdown` presents plain source until a renderer is installed, so\n * this component supplies one. Swap in any renderer that accepts\n * `MarkdownRendererProps` to change how answers are parsed and rendered.\n */\nexport function MarkdownRenderer({ source }: MarkdownRendererProps): React.JSX.Element {\n  return <ReactMarkdown remarkPlugins={[remarkGfm]}>{source}</ReactMarkdown>;\n}\n",
@@ -31,7 +31,7 @@
         "globals.css": "@import \"tailwindcss\";\n",
         "public/favicon.svg": "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 64 64\">\n  <rect width=\"64\" height=\"64\" fill=\"#fff\"/>\n  <circle cx=\"32\" cy=\"32\" r=\"20\" fill=\"#000\"/>\n</svg>\n",
         "README.md": "# AI Agent\n\nA small, customizable agent with a streaming chat UI and tool support.\n\n## What's included\n\n- Single assistant agent with streaming chat UI\n- Example calculator tool\n- Smoke eval for the agent and calculator\n- App-mode `Chat` component for real-time responses\n\n## Structure\n\n```\nagents/assistant.ts    Agent definition\ntools/calculator.ts    Example tool\nevals/assistant.eval.ts Agent smoke eval\napp/\n  api/ag-ui/route.ts    AG-UI endpoint\n  page.tsx             Chat interface\n```\n\n## Customize\n\n- Edit `agents/assistant.ts` to change the agent's identity, instructions, and suggestions.\n- Add or replace files in `tools/` to give the agent new capabilities.\n- Update `evals/assistant.eval.ts`, then run `npm run eval -- assistant`.\n- Edit `app/page.tsx` when you need to customize the chat UI.\n",
-        "tools/calculator.ts": "import { tool } from \"veryfront/tool\";\nimport { defineSchema } from \"veryfront/schemas\";\n\nexport default tool({\n  id: \"calculator\",\n  description: \"Perform arithmetic. For round, a is the value and b is the decimal places.\",\n  inputSchema: defineSchema((v) => v.object({\n    operation: v.enum([\"add\", \"subtract\", \"multiply\", \"divide\", \"round\"]),\n    a: v.number(),\n    b: v.number(),\n  }))(),\n  execute: ({ operation, a, b }) => {\n    const precision = Math.min(100, Math.max(0, Math.trunc(b)));\n\n    if (operation === \"divide\" && b === 0) {\n      throw new Error(\"Cannot divide by zero\");\n    }\n\n    if (operation === \"add\") return { result: a + b };\n    if (operation === \"subtract\") return { result: a - b };\n    if (operation === \"multiply\") return { result: a * b };\n    if (operation === \"round\") {\n      const offset = Math.sign(a) * Number.EPSILON * Math.max(1, Math.abs(a));\n      return { result: Number((a + offset).toFixed(precision)) };\n    }\n    return { result: a / b };\n  },\n});\n",
+        "tools/calculator.ts": "import { tool } from \"veryfront/tool\";\nimport { defineSchema } from \"veryfront/schemas\";\n\nexport default tool({\n  id: \"calculator\",\n  description:\n    \"Perform one arithmetic operation on two numbers. Use split to divide a money amount a into b shares that add up to it exactly.\",\n  inputSchema: defineSchema((v) =>\n    v.object({\n      operation: v.enum([\"add\", \"subtract\", \"multiply\", \"divide\", \"split\"]),\n      a: v.number(),\n      b: v.number(),\n    })\n  )(),\n  execute: ({ operation, a, b }) => {\n    if ((operation === \"divide\" || operation === \"split\") && b === 0) {\n      throw new Error(\"Cannot divide by zero\");\n    }\n\n    if (operation === \"split\") {\n      const parts = Math.max(1, Math.trunc(Math.abs(b)));\n      if (parts > 1000) throw new Error(\"Cannot split into more than 1000 shares\");\n\n      const cents = Math.round(a * 100);\n      const base = Math.trunc(cents / parts);\n      const remainder = Math.abs(cents - base * parts);\n      return {\n        result: Array.from(\n          { length: parts },\n          (_, index) => (base + (index < remainder ? Math.sign(cents) : 0)) / 100,\n        ),\n      };\n    }\n\n    if (operation === \"add\") return { result: a + b };\n    if (operation === \"subtract\") return { result: a - b };\n    if (operation === \"multiply\") return { result: a * b };\n    return { result: a / b };\n  },\n});\n",
         "tsconfig.json": "{\n  \"compilerOptions\": {\n    \"target\": \"ES2022\",\n    \"module\": \"ESNext\",\n    \"moduleResolution\": \"bundler\",\n    \"strict\": true,\n    \"jsx\": \"react-jsx\",\n    \"skipLibCheck\": true,\n    \"esModuleInterop\": true,\n    \"noEmit\": true,\n    \"allowImportingTsExtensions\": true,\n    \"paths\": {\n      \"@/*\": [\n        \"./*\"\n      ],\n      \"react-markdown@*\": [\n        \"./node_modules/react-markdown\"\n      ],\n      \"remark-gfm@*\": [\n        \"./node_modules/remark-gfm\"\n      ]\n    }\n  },\n  \"include\": [\n    \"**/*.ts\",\n    \"**/*.tsx\"\n  ],\n  \"exclude\": [\n    \"node_modules\"\n  ]\n}\n"
       }
     },

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "veryfront",
-  "version": "0.1.1211",
+  "version": "0.1.1212",
   "license": "Apache-2.0",
   "nodeModulesDir": "auto",
   "minimumDependencyAge": {

src/utils/version-constant.ts

@@ -1,4 +1,4 @@
 // Keep in sync with deno.json version.
 // scripts/release.ts updates this constant during releases.
 /** Shared version value. */
-export const VERSION = "0.1.1211";
+export const VERSION = "0.1.1212";