test: PPT quality eval baseline with svg-pptx route and machine gates (#1273)

.gitignore

@@ -24,6 +24,9 @@ target
 .scripts
 .direnv/
 
+__pycache__/
+*.pyc
+
 # Local dev files
 opencode-dev
 UPCOMING_CHANGELOG.md

packages/opencode/package.json

@@ -12,6 +12,8 @@
     "test:ci": "mkdir -p .artifacts/unit && bun test --timeout 30000 --reporter=junit --reporter-outfile=.artifacts/unit/junit.xml",
     "build": "bun run script/build.ts",
     "build:embedded-server": "bun run script/build-node.ts",
+    "office:eval": "bun run script/office-route-eval/eval.ts",
+    "ppt:eval": "bun run script/ppt-quality-eval/eval.ts",
     "route:inventory": "bun run script/route-inventory.ts",
     "fix-node-pty": "bun run script/fix-node-pty.ts",
     "dev": "bun run --conditions=browser ./src/index.ts",

packages/opencode/script/office-route-eval/README.md

@@ -0,0 +1,55 @@
+# Office Route Eval Baseline
+
+Local baseline for issue #1273. It compares the current OfficeCLI route with a
+Python + uv + skills route across three real office tasks:
+
+- `xlsx-dashboard`: CSV to Excel dashboard.
+- `docx-board-memo`: notes to Word board memo.
+- `pptx-pitch-deck`: brief to pitch deck.
+
+The judge never uses LibreOffice. It inspects OOXML zip contents, the raw
+opencode JSON event stream, command audit logs, artifact hashes, and route
+metadata.
+
+## Routes
+
+`officecli` means the current PawWork OfficeCLI skill route. The route must call
+`officecli` for the delivered Office artifact. Lightweight preprocessing is
+allowed only when it does not create the final `.xlsx`, `.docx`, or `.pptx`
+through Python Office libraries.
+
+`python` means Python + uv + local route skill. It must call `uv` and must not
+call `officecli`.
+
+Both routes fail if they call `libreoffice`, `soffice`, `lowriter`, `localc`, or
+`loffice`.
+
+## Run
+
+From `packages/opencode`:
+
+```bash
+bun run office:eval calibrate --model openai/gpt-5.4-mini --variant low
+bun run office:eval full --model openai/gpt-5.4-mini --variant low --rounds 3
+bun run office:eval report
+```
+
+`calibrate` runs `3 tasks x 2 routes x 1 round`. `full` runs all tasks and both
+routes for the requested rounds. Output lands in `script/office-route-eval/runs/`
+and is ignored by git.
+
+Each run contains:
+
+- `prompt.md`
+- `events.jsonl`
+- `stderr.log`
+- `run-summary.json`
+- `judge.json`
+- `artifacts/`
+
+## Replacement Bar
+
+This eval pack is enough to open a formal replacement PR only if the Python
+route passes all three task families in at least two of three rounds, has zero
+route-policy failures, and does not need more repair steps than the OfficeCLI
+route on the same tasks.

packages/opencode/script/office-route-eval/eval.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, test } from "bun:test"
+
+import { commandPolicyFailures, extractCommandsFromJsonl } from "./eval"
+
+describe("office route eval harness", () => {
+  test("extracts shell commands from opencode json events", () => {
+    const jsonl = [
+      JSON.stringify({
+        type: "tool_use",
+        part: {
+          tool: "bash",
+          state: {
+            status: "completed",
+            input: { command: "officecli create artifacts/out.xlsx", description: "create workbook" },
+          },
+        },
+      }),
+      JSON.stringify({ type: "text", part: { text: "done" } }),
+    ].join("\n")
+
+    const audit = extractCommandsFromJsonl(jsonl)
+    expect(audit.commands).toEqual([
+      {
+        tool: "bash",
+        command: "officecli create artifacts/out.xlsx",
+        description: "create workbook",
+        status: "completed",
+      },
+    ])
+    expect(audit.eventCounts.tool_use).toBe(1)
+  })
+
+  test("enforces python route tool boundary", () => {
+    expect(commandPolicyFailures("python", [{ tool: "bash", command: "uv run python build.py" }])).toEqual([])
+    expect(commandPolicyFailures("python", [{ tool: "bash", command: "officecli create out.xlsx" }])).toContain(
+      "Python route did not call uv.",
+    )
+  })
+
+  test("enforces officecli route tool boundary", () => {
+    expect(commandPolicyFailures("officecli", [{ tool: "bash", command: "officecli create out.docx" }])).toEqual([])
+    expect(commandPolicyFailures("officecli", [{ tool: "bash", command: "uv run python build.py" }])).toContain(
+      "OfficeCLI route did not call officecli.",
+    )
+  })
+})