diff --git a/.gitignore b/.gitignore index b216d6712..619eff16f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ target .scripts .direnv/ +__pycache__/ +*.pyc + # Local dev files opencode-dev UPCOMING_CHANGELOG.md diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 676a8a1dc..38cfbbfd8 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -12,6 +12,8 @@ "test:ci": "mkdir -p .artifacts/unit && bun test --timeout 30000 --reporter=junit --reporter-outfile=.artifacts/unit/junit.xml", "build": "bun run script/build.ts", "build:embedded-server": "bun run script/build-node.ts", + "office:eval": "bun run script/office-route-eval/eval.ts", + "ppt:eval": "bun run script/ppt-quality-eval/eval.ts", "route:inventory": "bun run script/route-inventory.ts", "fix-node-pty": "bun run script/fix-node-pty.ts", "dev": "bun run --conditions=browser ./src/index.ts", diff --git a/packages/opencode/script/office-route-eval/README.md b/packages/opencode/script/office-route-eval/README.md new file mode 100644 index 000000000..4813623c7 --- /dev/null +++ b/packages/opencode/script/office-route-eval/README.md @@ -0,0 +1,55 @@ +# Office Route Eval Baseline + +Local baseline for issue #1273. It compares the current OfficeCLI route with a +Python + uv + skills route across three real office tasks: + +- `xlsx-dashboard`: CSV to Excel dashboard. +- `docx-board-memo`: notes to Word board memo. +- `pptx-pitch-deck`: brief to pitch deck. + +The judge never uses LibreOffice. It inspects OOXML zip contents, the raw +opencode JSON event stream, command audit logs, artifact hashes, and route +metadata. + +## Routes + +`officecli` means the current PawWork OfficeCLI skill route. The route must call +`officecli` for the delivered Office artifact. Lightweight preprocessing is +allowed only when it does not create the final `.xlsx`, `.docx`, or `.pptx` +through Python Office libraries. + +`python` means Python + uv + local route skill. It must call `uv` and must not +call `officecli`. + +Both routes fail if they call `libreoffice`, `soffice`, `lowriter`, `localc`, or +`loffice`. + +## Run + +From `packages/opencode`: + +```bash +bun run office:eval calibrate --model openai/gpt-5.4-mini --variant low +bun run office:eval full --model openai/gpt-5.4-mini --variant low --rounds 3 +bun run office:eval report +``` + +`calibrate` runs `3 tasks x 2 routes x 1 round`. `full` runs all tasks and both +routes for the requested rounds. Output lands in `script/office-route-eval/runs/` +and is ignored by git. + +Each run contains: + +- `prompt.md` +- `events.jsonl` +- `stderr.log` +- `run-summary.json` +- `judge.json` +- `artifacts/` + +## Replacement Bar + +This eval pack is enough to open a formal replacement PR only if the Python +route passes all three task families in at least two of three rounds, has zero +route-policy failures, and does not need more repair steps than the OfficeCLI +route on the same tasks. diff --git a/packages/opencode/script/office-route-eval/eval.test.ts b/packages/opencode/script/office-route-eval/eval.test.ts new file mode 100644 index 000000000..fca137cb7 --- /dev/null +++ b/packages/opencode/script/office-route-eval/eval.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test" + +import { commandPolicyFailures, extractCommandsFromJsonl } from "./eval" + +describe("office route eval harness", () => { + test("extracts shell commands from opencode json events", () => { + const jsonl = [ + JSON.stringify({ + type: "tool_use", + part: { + tool: "bash", + state: { + status: "completed", + input: { command: "officecli create artifacts/out.xlsx", description: "create workbook" }, + }, + }, + }), + JSON.stringify({ type: "text", part: { text: "done" } }), + ].join("\n") + + const audit = extractCommandsFromJsonl(jsonl) + expect(audit.commands).toEqual([ + { + tool: "bash", + command: "officecli create artifacts/out.xlsx", + description: "create workbook", + status: "completed", + }, + ]) + expect(audit.eventCounts.tool_use).toBe(1) + }) + + test("enforces python route tool boundary", () => { + expect(commandPolicyFailures("python", [{ tool: "bash", command: "uv run python build.py" }])).toEqual([]) + expect(commandPolicyFailures("python", [{ tool: "bash", command: "officecli create out.xlsx" }])).toContain( + "Python route did not call uv.", + ) + }) + + test("enforces officecli route tool boundary", () => { + expect(commandPolicyFailures("officecli", [{ tool: "bash", command: "officecli create out.docx" }])).toEqual([]) + expect(commandPolicyFailures("officecli", [{ tool: "bash", command: "uv run python build.py" }])).toContain( + "OfficeCLI route did not call officecli.", + ) + }) +}) diff --git a/packages/opencode/script/office-route-eval/eval.ts b/packages/opencode/script/office-route-eval/eval.ts new file mode 100644 index 000000000..fe2f97efb --- /dev/null +++ b/packages/opencode/script/office-route-eval/eval.ts @@ -0,0 +1,744 @@ +import { createHash } from "node:crypto" +import { copyFile, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises" +import path from "node:path" +import { BlobReader, TextWriter, ZipReader } from "@zip.js/zip.js" + +type RouteID = "officecli" | "python" +type TaskID = "xlsx-dashboard" | "docx-board-memo" | "pptx-pitch-deck" + +type CommandAudit = { + tool: string + command: string + description?: string + status?: string +} + +type ArtifactSummary = { + path: string + exists: boolean + size: number + sha256?: string +} + +type RunSummary = { + schemaVersion: 1 + runId: string + taskId: TaskID + routeId: RouteID + round: number + model: string + variant?: string + startedAt: string + completedAt: string + durationMs: number + exitCode: number | null + workDir: string + artifactPath: string + officecli: { + manifestVersion: string + binaryPath: string + binaryVersion?: string + } + python: { + uvVersion?: string + pythonVersion?: string + } + commands: CommandAudit[] + artifacts: ArtifactSummary[] + eventCounts: Record +} + +type JudgeResult = { + schemaVersion: 1 + runId: string + taskId: TaskID + routeId: RouteID + passed: boolean + score: number + failures: string[] + warnings: string[] + metrics: Record +} + +type TaskSpec = { + id: TaskID + artifact: string + fixture: string + prompt: string + requiredText: string[] +} + +const evalRoot = import.meta.dir +const packageRoot = path.resolve(evalRoot, "../..") +const repoRoot = path.resolve(packageRoot, "../..") +const runsRoot = path.join(evalRoot, "runs") +const toolsDir = path.join(repoRoot, "packages/desktop-electron/resources/tools") +const officeCliPath = path.join(toolsDir, process.platform === "win32" ? "officecli.exe" : "officecli") +const officeManifestPath = path.join(repoRoot, "packages/desktop-electron/bundled-tools.json") + +const tasks: Record = { + "xlsx-dashboard": { + id: "xlsx-dashboard", + artifact: "sales-dashboard.xlsx", + fixture: "sales-2026.csv", + requiredText: [ + "Orion Analytics Sales Dashboard", + "Raw Data", + "Dashboard", + "Regional Summary", + "Total Revenue", + "Gross Margin Rate", + "Customers", + "Top Region", + "North", + "South", + "West", + ], + prompt: [ + "Create an Excel workbook named sales-dashboard.xlsx from the attached sales-2026.csv.", + "The workbook must have exactly these user-facing sheets: Raw Data, Dashboard, Regional Summary.", + "Import every CSV row into Raw Data.", + "Dashboard must contain the title 'Orion Analytics Sales Dashboard' and KPI labels Total Revenue, Gross Margin Rate, Customers, Top Region.", + "Use formulas for computed dashboard and regional summary numbers when the format supports formulas.", + "Add at least one column or bar chart based on the regional or monthly summary.", + "Freeze the Raw Data header row and set readable explicit column widths.", + "Write the final artifact to ./artifacts/sales-dashboard.xlsx.", + ].join("\n"), + }, + "docx-board-memo": { + id: "docx-board-memo", + artifact: "board-memo.docx", + fixture: "board-notes.md", + requiredText: [ + "Orion Assist Board Memo", + "Executive Summary", + "Decision Needed", + "Evidence", + "Risks", + "Next Steps", + "Metric", + "Current", + "Target", + "Status", + ], + prompt: [ + "Create a Word document named board-memo.docx from the attached board-notes.md.", + "The document title must be 'Orion Assist Board Memo'.", + "Use these section headings: Executive Summary, Decision Needed, Evidence, Risks, Next Steps.", + "Include a table with headers Metric, Current, Target, Status and fill it from the source notes.", + "Add a footer with a live page-number field, not static text.", + "Use explicit heading and body styling; avoid empty paragraphs as spacing.", + "Write the final artifact to ./artifacts/board-memo.docx.", + ].join("\n"), + }, + "pptx-pitch-deck": { + id: "pptx-pitch-deck", + artifact: "orion-assist-pitch.pptx", + fixture: "growth-brief.md", + requiredText: [ + "Orion Assist", + "Problem", + "Solution", + "Market", + "Go-To-Market", + "Ask", + "$2.4M", + "18 months", + ], + prompt: [ + "Create a six-slide PowerPoint deck named orion-assist-pitch.pptx from the attached growth-brief.md.", + "Use exactly these slide titles: Orion Assist, Problem, Solution, Market, Go-To-Market, Ask.", + "Every slide must have explicit title and body font sizes.", + "Slides 2 through 6 must include speaker notes.", + "Include at least one chart, preferably on the Market or Go-To-Market slide.", + "Avoid placeholder text and do not leave a bullet-only deck.", + "Write the final artifact to ./artifacts/orion-assist-pitch.pptx.", + ].join("\n"), + }, +} + +function usage() { + console.log(`Usage: + bun run office:eval calibrate --model ollama-cloud/deepseek-v4-flash --variant low + bun run office:eval full --model ollama-cloud/deepseek-v4-flash --variant low --rounds 3 + bun run office:eval full --model ollama-cloud/deepseek-v4-flash --variant low --start-round 2 --rounds 3 + bun run office:eval run --task xlsx-dashboard --route officecli --round 1 --model ollama-cloud/deepseek-v4-flash --variant low + bun run office:eval judge --run script/office-route-eval/runs/ + bun run office:eval report`) +} + +function parseArgs(argv: string[]) { + const out: Record = {} + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] + if (!arg.startsWith("--")) continue + const key = arg.slice(2) + const next = argv[i + 1] + if (!next || next.startsWith("--")) { + out[key] = true + continue + } + out[key] = next + i += 1 + } + return out +} + +function assertTask(value: unknown): TaskID { + if (value === "xlsx-dashboard" || value === "docx-board-memo" || value === "pptx-pitch-deck") return value + throw new Error(`Unknown task: ${String(value)}`) +} + +function assertRoute(value: unknown): RouteID { + if (value === "officecli" || value === "python") return value + throw new Error(`Unknown route: ${String(value)}`) +} + +function decodeXml(value: string) { + return value + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, "&") +} + +function textFromXml(xml: string) { + return [...xml.matchAll(/<(?:[a-zA-Z0-9]+:)?t(?:\s[^>]*)?>([\s\S]*?)<\/(?:[a-zA-Z0-9]+:)?t>/g)] + .map((match) => decodeXml(match[1])) + .join(" ") +} + +function countMatches(value: string, pattern: RegExp) { + return [...value.matchAll(pattern)].length +} + +function hasAny(value: string, needles: string[]) { + const lower = value.toLowerCase() + return needles.some((needle) => lower.includes(needle.toLowerCase())) +} + +function hasPlaceholderText(value: string) { + return /\{\{||lorem|xxxx|\$xxx\$/.test(value) +} + +async function readZipEntries(file: string) { + const bytes = await readFile(file) + const arrayBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer + const reader = new ZipReader(new BlobReader(new Blob([arrayBuffer]))) + try { + const entries = await reader.getEntries() + const result = new Map() + for (const entry of entries) { + if (entry.directory || !entry.getData) continue + if (!/\.(xml|rels)$/i.test(entry.filename) && !entry.filename.endsWith("[Content_Types].xml")) continue + result.set(entry.filename, await entry.getData(new TextWriter())) + } + return result + } finally { + await reader.close() + } +} + +async function sha256File(file: string) { + const bytes = await readFile(file) + return createHash("sha256").update(bytes).digest("hex") +} + +async function commandOutput(command: string, args: string[]) { + const proc = Bun.spawn([command, ...args], { stdout: "pipe", stderr: "pipe" }) + const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]) + return code === 0 ? stdout.trim() : stderr.trim() || stdout.trim() +} + +async function officeManifestVersion() { + const raw = await readFile(officeManifestPath, "utf8") + return JSON.parse(raw).officecli.version as string +} + +function routeConfig(route: RouteID) { + const skillPath = + route === "officecli" + ? path.join(evalRoot, "route-skills/officecli-eval-policy") + : path.join(evalRoot, "route-skills/python-office-eval") + return { + skills: { + paths: [skillPath], + }, + permission: { + bash: "allow", + read: "allow", + write: "allow", + edit: "allow", + skill: "allow", + }, + } +} + +async function prepareRunDir(task: TaskSpec, route: RouteID, round: number) { + const stamp = new Date().toISOString().replace(/[:.]/g, "-") + const runId = `${stamp}-${task.id}-${route}-r${round}` + const runDir = path.join(runsRoot, runId) + const workDir = path.join(runDir, "workspace") + const artifactDir = path.join(workDir, "artifacts") + const inputDir = path.join(workDir, "input") + await mkdir(artifactDir, { recursive: true }) + await mkdir(inputDir, { recursive: true }) + await copyFile(path.join(evalRoot, "fixtures", task.fixture), path.join(inputDir, task.fixture)) + if (route === "python") { + await copyFile(path.join(evalRoot, "route-templates/python/pyproject.toml"), path.join(workDir, "pyproject.toml")) + } + return { runId, runDir, workDir, artifactDir, inputDir } +} + +function buildPrompt(task: TaskSpec, route: RouteID, runDir: string, workDir: string) { + const artifact = path.join(workDir, "artifacts", task.artifact) + const routeLine = + route === "officecli" + ? "Use the current PawWork OfficeCLI route. You must use officecli for the final Office artifact." + : "Use the Python + uv + skills route. You must use uv and Python libraries. Do not call officecli." + return [ + "# Office route eval task", + "", + routeLine, + "Do not use LibreOffice, soffice, lowriter, localc, or loffice.", + "Work only inside the current working directory.", + "Use the attached source file and the same source path under ./input/.", + "", + task.prompt, + "", + "After creating the artifact, write ./artifacts/artifact-summary.json with:", + JSON.stringify( + { + artifact: artifact, + route, + task: task.id, + commandsUsed: ["short list of important commands"], + limitations: [], + }, + null, + 2, + ), + "", + "Do not claim success unless the target artifact exists.", + `Write only under this working directory: ${workDir}`, + ].join("\n") +} + +export function extractCommandsFromJsonl(jsonl: string): { commands: CommandAudit[]; eventCounts: Record } { + const commands: CommandAudit[] = [] + const eventCounts: Record = {} + for (const line of jsonl.split(/\r?\n/)) { + if (!line.trim()) continue + let event: any + try { + event = JSON.parse(line) + } catch { + continue + } + eventCounts[event.type] = (eventCounts[event.type] ?? 0) + 1 + const part = event.part + if (event.type !== "tool_use" || !part?.state?.input) continue + const tool = String(part.tool ?? "") + if (!["bash", "cmd", "powershell", "pwsh"].includes(tool)) continue + const input = part.state.input + if (typeof input.command !== "string") continue + commands.push({ + tool, + command: input.command, + description: typeof input.description === "string" ? input.description : undefined, + status: typeof part.state.status === "string" ? part.state.status : undefined, + }) + } + return { commands, eventCounts } +} + +export function commandPolicyFailures(route: RouteID, commands: CommandAudit[]) { + const failures: string[] = [] + const joined = commands.map((item) => item.command).join("\n").toLowerCase() + if (/\b(libreoffice|soffice|lowriter|localc|loffice)\b/.test(joined)) { + failures.push("Route used LibreOffice or a LibreOffice alias.") + } + if (route === "python") { + if (!/\buv\b/.test(joined)) failures.push("Python route did not call uv.") + if (/\bofficecli\b/.test(joined)) failures.push("Python route called officecli.") + } + if (route === "officecli") { + if (!/\bofficecli\b/.test(joined)) failures.push("OfficeCLI route did not call officecli.") + if (/\buv\b/.test(joined)) failures.push("OfficeCLI route called uv.") + if (/\b(openpyxl|python-docx|python_pptx|python-pptx|from pptx|from docx|load_workbook|workbook\()/i.test(joined)) { + failures.push("OfficeCLI route appears to create the final artifact through Python Office libraries.") + } + } + return failures +} + +async function artifactSummaries(artifactPath: string): Promise { + const info = await stat(artifactPath).catch(() => undefined) + if (!info?.isFile()) return [{ path: artifactPath, exists: false, size: 0 }] + return [{ path: artifactPath, exists: true, size: info.size, sha256: await sha256File(artifactPath) }] +} + +async function findByName(root: string, basename: string, depth = 5): Promise { + const found: string[] = [] + async function walk(dir: string, remaining: number) { + if (remaining < 0) return + const entries = await readdir(dir, { withFileTypes: true }).catch(() => []) + for (const entry of entries) { + const full = path.join(dir, entry.name) + if (entry.isFile() && entry.name === basename) found.push(full) + if (entry.isDirectory()) await walk(full, remaining - 1) + } + } + await walk(root, depth) + return found +} + +async function runOne(taskId: TaskID, routeId: RouteID, round: number, model: string, variant?: string) { + const task = tasks[taskId] + const { runId, runDir, workDir } = await prepareRunDir(task, routeId, round) + const startedAt = new Date() + const prompt = buildPrompt(task, routeId, runDir, workDir) + const promptPath = path.join(runDir, "prompt.md") + await writeFile(promptPath, prompt) + + const args = [ + "--cwd", + packageRoot, + "src/index.ts", + "run", + // --file is array-typed and would swallow a following positional prompt; keep scalar flags after it. + "--file", + path.join(workDir, "input", task.fixture), + "--format", + "json", + "--model", + model, + "--dir", + workDir, + "--dangerously-skip-permissions", + ] + if (variant) args.push("--variant", variant) + args.push(prompt) + + const env = { + ...process.env, + OPENCODE_DISABLE_EXTERNAL_SKILLS: "1", + OPENCODE_CONFIG_CONTENT: JSON.stringify(routeConfig(routeId)), + OFFICECLI_SKIP_UPDATE: "1", + PATH: routeId === "officecli" ? `${toolsDir}${path.delimiter}${process.env.PATH ?? ""}` : process.env.PATH ?? "", + } + + const proc = Bun.spawn(["bun", ...args], { + cwd: repoRoot, + env, + stdout: "pipe", + stderr: "pipe", + }) + const timeoutMs = Number(process.env.OFFICE_ROUTE_EVAL_TIMEOUT_MS ?? 10 * 60 * 1000) + const killer = setTimeout(() => proc.kill(), timeoutMs) + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]).finally(() => clearTimeout(killer)) + + await writeFile(path.join(runDir, "events.jsonl"), stdout) + await writeFile(path.join(runDir, "stderr.log"), stderr) + + const { commands, eventCounts } = extractCommandsFromJsonl(stdout) + const artifactPath = path.join(workDir, "artifacts", task.artifact) + const completedAt = new Date() + const summary: RunSummary = { + schemaVersion: 1, + runId, + taskId, + routeId, + round, + model, + variant, + startedAt: startedAt.toISOString(), + completedAt: completedAt.toISOString(), + durationMs: completedAt.getTime() - startedAt.getTime(), + exitCode, + workDir, + artifactPath, + officecli: { + manifestVersion: await officeManifestVersion(), + binaryPath: officeCliPath, + binaryVersion: await commandOutput(officeCliPath, ["--version"]).catch((error) => String(error)), + }, + python: { + uvVersion: await commandOutput("uv", ["--version"]).catch((error) => String(error)), + pythonVersion: await commandOutput("python3", ["--version"]).catch((error) => String(error)), + }, + commands, + artifacts: await artifactSummaries(artifactPath), + eventCounts, + } + await writeFile(path.join(runDir, "run-summary.json"), JSON.stringify(summary, null, 2)) + const judge = await judgeRun(runDir) + console.log(`${judge.passed ? "PASS" : "FAIL"} ${runId} score=${judge.score}`) + if (judge.failures.length) console.log(judge.failures.map((item) => ` - ${item}`).join("\n")) +} + +function addRequiredTextFailures(failures: string[], text: string, required: string[]) { + for (const item of required) { + if (!text.toLowerCase().includes(item.toLowerCase())) failures.push(`Missing required text: ${item}`) + } +} + +async function judgeXlsx(summary: RunSummary, task: TaskSpec, failures: string[], warnings: string[], metrics: JudgeResult["metrics"]) { + const zip = await readZipEntries(summary.artifactPath) + const names = [...zip.keys()] + const allXml = [...zip.values()].join("\n") + const sharedText = textFromXml(zip.get("xl/sharedStrings.xml") ?? "") + const combined = `${allXml}\n${sharedText}` + metrics.zipEntries = names.length + metrics.chartCount = names.filter((name) => name.startsWith("xl/charts/chart")).length + metrics.formulaCount = countMatches(allXml, /]*)?>[\s\S]*?<\/f>/g) + if (!zip.has("[Content_Types].xml") || !zip.has("xl/workbook.xml")) failures.push("Invalid xlsx package structure.") + addRequiredTextFailures(failures, combined, task.requiredText) + if (Number(metrics.chartCount) < 1) failures.push("XLSX has no chart XML.") + if (Number(metrics.formulaCount) < 3) failures.push("XLSX has fewer than three formulas.") + if (!hasAny(allXml, ["SUM(", "SUMIF", "SUMIFS", "AVERAGE", "SUBTOTAL"])) { + failures.push("XLSX formulas do not include recognizable summary calculations.") + } + for (const token of ["#REF!", "#DIV/0!", "#VALUE!", "#NAME?", "#N/A"]) { + if (combined.includes(token)) failures.push(`XLSX contains formula error token ${token}.`) + } + if (hasPlaceholderText(textFromXml(combined))) failures.push("XLSX contains placeholder-like text.") + if (!zip.has("xl/worksheets/sheet1.xml")) warnings.push("XLSX sheet1.xml missing; workbook may use unusual sheet layout.") +} + +async function judgeDocx(summary: RunSummary, task: TaskSpec, failures: string[], _warnings: string[], metrics: JudgeResult["metrics"]) { + const zip = await readZipEntries(summary.artifactPath) + const documentXml = zip.get("word/document.xml") ?? "" + const footerXml = [...zip.entries()] + .filter(([name]) => /^word\/footer\d+\.xml$/.test(name)) + .map(([, value]) => value) + .join("\n") + const text = textFromXml(`${documentXml}\n${footerXml}`) + metrics.paragraphCount = countMatches(documentXml, /]/g) + metrics.tableCount = countMatches(documentXml, /]/g) + metrics.headingStyleCount = countMatches(documentXml, /]+w:val="Heading[123]"/g) + metrics.footerCount = countMatches(footerXml, /]/g) + if (!zip.has("[Content_Types].xml") || !zip.has("word/document.xml")) failures.push("Invalid docx package structure.") + addRequiredTextFailures(failures, text, task.requiredText) + if (Number(metrics.tableCount) < 1) failures.push("DOCX has no table.") + if (Number(metrics.headingStyleCount) < 3) failures.push("DOCX has fewer than three Heading styles.") + if (!footerXml || !footerXml.includes("fldChar")) failures.push("DOCX footer does not contain a live page-number field.") + if (hasPlaceholderText(text)) failures.push("DOCX contains placeholder-like text.") +} + +async function judgePptx(summary: RunSummary, task: TaskSpec, failures: string[], warnings: string[], metrics: JudgeResult["metrics"]) { + const zip = await readZipEntries(summary.artifactPath) + const slideEntries = [...zip.entries()].filter(([name]) => /^ppt\/slides\/slide\d+\.xml$/.test(name)) + const notesEntries = [...zip.keys()].filter((name) => /^ppt\/notesSlides\/notesSlide\d+\.xml$/.test(name)) + const chartEntries = [...zip.keys()].filter((name) => name.startsWith("ppt/charts/chart")) + const slideXml = slideEntries.map(([, value]) => value).join("\n") + const text = textFromXml(slideXml) + const fontSizes = [...slideXml.matchAll(/]+val="(\d+)"/g)].map((match) => Number(match[1])) + metrics.slideCount = slideEntries.length + metrics.notesCount = notesEntries.length + metrics.chartCount = chartEntries.length + metrics.maxFontCentipoints = fontSizes.length ? Math.max(...fontSizes) : 0 + metrics.minFontCentipoints = fontSizes.length ? Math.min(...fontSizes) : 0 + if (!zip.has("[Content_Types].xml") || !zip.has("ppt/presentation.xml")) failures.push("Invalid pptx package structure.") + if (slideEntries.length !== 6) failures.push(`PPTX expected exactly 6 slides, found ${slideEntries.length}.`) + addRequiredTextFailures(failures, text, task.requiredText) + if (notesEntries.length < 5) failures.push("PPTX has fewer than five notes slides.") + if (chartEntries.length < 1) failures.push("PPTX has no chart XML.") + if (!fontSizes.length) failures.push("PPTX has no explicit run font sizes.") + if (fontSizes.length && Math.max(...fontSizes) < 3600) failures.push("PPTX has no title-sized text at or above 36pt.") + if (fontSizes.length && Math.min(...fontSizes) < 1000) warnings.push("PPTX includes text below 10pt.") + if (hasPlaceholderText(text)) failures.push("PPTX contains placeholder-like text.") +} + +async function judgeArtifact(summary: RunSummary, failures: string[], warnings: string[], metrics: JudgeResult["metrics"]) { + const task = tasks[summary.taskId] + const artifact = summary.artifacts[0] + if (!artifact?.exists) { + const candidates = await findByName(summary.workDir, path.basename(summary.artifactPath)) + failures.push( + candidates.length + ? `Target artifact does not exist at required path; found at ${candidates.join(", ")}.` + : "Target artifact does not exist.", + ) + return + } + metrics.artifactSize = artifact.size + metrics.artifactSha256 = artifact.sha256 ?? "" + const artifactSummaryPath = path.join(path.dirname(summary.artifactPath), "artifact-summary.json") + if (!(await stat(artifactSummaryPath).catch(() => undefined))?.isFile()) { + failures.push("artifact-summary.json is missing.") + } + if (summary.taskId === "xlsx-dashboard") await judgeXlsx(summary, task, failures, warnings, metrics) + if (summary.taskId === "docx-board-memo") await judgeDocx(summary, task, failures, warnings, metrics) + if (summary.taskId === "pptx-pitch-deck") await judgePptx(summary, task, failures, warnings, metrics) +} + +export async function judgeRun(runDir: string): Promise { + const summary = JSON.parse(await readFile(path.join(runDir, "run-summary.json"), "utf8")) as RunSummary + const failures = commandPolicyFailures(summary.routeId, summary.commands) + const warnings: string[] = [] + const metrics: JudgeResult["metrics"] = { + commandCount: summary.commands.length, + exitCode: summary.exitCode ?? -1, + durationMs: summary.durationMs, + } + if (summary.exitCode !== 0) failures.push(`opencode run exited with ${summary.exitCode}.`) + await judgeArtifact(summary, failures, warnings, metrics).catch((error) => { + failures.push(`Artifact judge failed: ${error instanceof Error ? error.message : String(error)}`) + }) + const score = Math.max(0, Math.min(100, 100 - failures.length * 18 - warnings.length * 4)) + const result: JudgeResult = { + schemaVersion: 1, + runId: summary.runId, + taskId: summary.taskId, + routeId: summary.routeId, + passed: failures.length === 0, + score, + failures, + warnings, + metrics, + } + await writeFile(path.join(runDir, "judge.json"), JSON.stringify(result, null, 2)) + return result +} + +async function matrix(startRound: number, rounds: number, model: string, variant?: string) { + for (let round = startRound; round <= rounds; round++) { + for (const taskId of Object.keys(tasks) as TaskID[]) { + for (const routeId of ["officecli", "python"] as RouteID[]) { + await runOne(taskId, routeId, round, model, variant) + } + } + } +} + +async function report() { + await mkdir(runsRoot, { recursive: true }) + const rows: { judge: JudgeResult; summary: RunSummary }[] = [] + for (const name of (await readdir(runsRoot)).sort()) { + const dir = path.join(runsRoot, name) + if (!(await stat(dir).catch(() => undefined))?.isDirectory()) continue + const judgePath = path.join(dir, "judge.json") + const summaryPath = path.join(dir, "run-summary.json") + if (!(await stat(judgePath).catch(() => undefined))?.isFile()) continue + const judge = JSON.parse(await readFile(judgePath, "utf8")) as JudgeResult + const summary = JSON.parse(await readFile(summaryPath, "utf8")) as RunSummary + rows.push({ judge, summary }) + } + rows.sort((a, b) => a.summary.round - b.summary.round || a.judge.taskId.localeCompare(b.judge.taskId) || a.judge.routeId.localeCompare(b.judge.routeId)) + const formatFailures = (judge: JudgeResult, summary: RunSummary) => + judge.failures + .map((failure) => failure.replaceAll(`${summary.workDir}/`, "./").replaceAll(evalRoot, "")) + .join("; ") + const detailRows = rows.map( + ({ judge, summary }) => + `| ${judge.taskId} | ${judge.routeId} | ${summary.round} | ${judge.passed ? "pass" : "fail"} | ${judge.score} | ${Math.round(summary.durationMs / 1000)} | ${summary.commands.length} | ${formatFailures(judge, summary)} |`, + ) + const aggregateRows: string[] = [] + for (const taskId of Object.keys(tasks) as TaskID[]) { + for (const routeId of ["officecli", "python"] as RouteID[]) { + const subset = rows.filter((row) => row.judge.taskId === taskId && row.judge.routeId === routeId) + if (!subset.length) continue + const passes = subset.filter((row) => row.judge.passed).length + const seconds = subset.map((row) => Math.round(row.summary.durationMs / 1000)).toSorted((a, b) => a - b) + const commands = subset.map((row) => row.summary.commands.length).toSorted((a, b) => a - b) + const median = (values: number[]) => values[Math.floor(values.length / 2)] ?? 0 + aggregateRows.push( + `| ${taskId} | ${routeId} | ${passes}/${subset.length} | ${median(seconds)} | ${median(commands)} |`, + ) + } + } + const taskIds = Object.keys(tasks) + const pythonByTask = taskIds.map((taskId) => { + const subset = rows.filter((row) => row.judge.taskId === taskId && row.judge.routeId === "python") + return subset.filter((row) => row.judge.passed).length + }) + const replacementReady = pythonByTask.length === 3 && pythonByTask.every((passes) => passes >= 2) + const verdict = replacementReady + ? "Python route cleared the replacement bar in this run set. A formal replacement PR is worth opening after a final review of failures, route policy, and artifact samples." + : `Do not open a formal OfficeCLI replacement PR yet: the Python route passed ${pythonByTask.map((passes, index) => `${passes} of the ${taskIds[index]} runs`).join(", ")}, below the bar of 2+ passes on all three tasks.` + const body = [ + "# Office Route Eval Report", + "", + "## Verdict", + "", + verdict, + "", + "Next smallest boundary: keep this eval harness as the baseline, then harden only the Python pptx skill/template so generated decks set run-level font sizes and pass the existing pptx judge in at least 2/3 rounds.", + "", + "## Aggregate", + "", + "| Task | Route | Passes | Median Seconds | Median Commands |", + "|---|---:|---:|---:|---:|", + ...(aggregateRows.length ? aggregateRows : ["| n/a | n/a | n/a | n/a | n/a |"]), + "", + "## Runs", + "", + "| Task | Route | Round | Result | Score | Seconds | Commands | Failures |", + "|---|---:|---:|---|---:|---:|---:|---|", + ...(detailRows.length ? detailRows : ["| n/a | n/a | n/a | n/a | n/a | n/a | n/a | No runs yet |"]), + "", + "Replacement PR bar: Python route must pass all three task families in at least two of three rounds with zero route-policy failures.", + ].join("\n") + const reportPath = path.join(evalRoot, "report.md") + await writeFile(reportPath, body) + console.log(reportPath) +} + +async function main() { + const [command, ...rest] = process.argv.slice(2) + const args = parseArgs(rest) + await mkdir(runsRoot, { recursive: true }) + if (!command || command === "help") { + usage() + return + } + if (command === "run") { + await runOne( + assertTask(args.task), + assertRoute(args.route), + Number(args.round ?? 1), + String(args.model ?? "ollama-cloud/deepseek-v4-flash"), + typeof args.variant === "string" ? args.variant : undefined, + ) + return + } + if (command === "calibrate") { + await matrix(1, 1, String(args.model ?? "ollama-cloud/deepseek-v4-flash"), typeof args.variant === "string" ? args.variant : undefined) + await report() + return + } + if (command === "full") { + await matrix( + Number(args["start-round"] ?? 1), + Number(args.rounds ?? 3), + String(args.model ?? "ollama-cloud/deepseek-v4-flash"), + typeof args.variant === "string" ? args.variant : undefined, + ) + await report() + return + } + if (command === "judge") { + const runDir = String(args.run ?? "") + if (!runDir) throw new Error("--run is required") + console.log(JSON.stringify(await judgeRun(path.resolve(packageRoot, runDir)), null, 2)) + return + } + if (command === "report") { + await report() + return + } + throw new Error(`Unknown command: ${command}`) +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.stack || error.message : String(error)) + process.exitCode = 1 + }) +} diff --git a/packages/opencode/script/office-route-eval/fixtures/board-notes.md b/packages/opencode/script/office-route-eval/fixtures/board-notes.md new file mode 100644 index 000000000..e5da44ad9 --- /dev/null +++ b/packages/opencode/script/office-route-eval/fixtures/board-notes.md @@ -0,0 +1,36 @@ +# Orion Assist Board Notes + +Audience: board operating committee. +Decision: approve a scoped beta for Orion Assist, an AI workflow companion for revenue operations teams. + +## Current Situation + +- Sales operations teams spend 9.5 hours per week cleaning CRM handoff data. +- Pilot users completed renewal-prep packets 38% faster with the prototype. +- The prototype reduced missed follow-up tasks from 14% to 5% across 42 sampled accounts. +- Support escalations stayed flat during the pilot. + +## Metrics + +| Metric | Current | Target | Status | +|---|---:|---:|---| +| Renewal packet prep time | 9.5 hours/week | 6.0 hours/week | Ahead | +| Missed follow-up rate | 14% | 7% | Ahead | +| Pilot active teams | 8 | 15 | Watch | +| Support escalations | Flat | Flat | On track | + +## Recommendation + +Approve a beta with 15 teams for one quarter. Keep the beta behind an admin flag. + +## Risks + +- CRM field mappings differ across enterprise accounts. +- Workflow suggestions need audit trails before broad rollout. +- Data export permissions must stay tenant-scoped. + +## Next Steps + +1. Confirm beta account list by June 19, 2026. +2. Add audit-log export before beta start. +3. Review beta metrics after four weeks. diff --git a/packages/opencode/script/office-route-eval/fixtures/growth-brief.md b/packages/opencode/script/office-route-eval/fixtures/growth-brief.md new file mode 100644 index 000000000..49c05030d --- /dev/null +++ b/packages/opencode/script/office-route-eval/fixtures/growth-brief.md @@ -0,0 +1,37 @@ +# Orion Assist Growth Brief + +Product: Orion Assist +Positioning: AI workflow companion for revenue operations teams. +Audience: seed extension investors. + +## Problem + +Revenue operations teams lose time moving between CRM records, spreadsheets, customer emails, and renewal notes. The cost is not just manual work; missed follow-up tasks create renewal risk. + +## Solution + +Orion Assist watches the work context, drafts renewal-prep packets, flags missing account data, and writes an auditable task trail back to the CRM. + +## Market + +- Initial segment: B2B SaaS revenue operations teams with 50 to 500 account owners. +- Serviceable market: 24,000 teams in North America and Western Europe. +- Initial ACV: $18,000. +- Three-year obtainable revenue target: $28M ARR. + +## Go-To-Market + +- Start with design partners from the current beta waitlist. +- Convert teams after a 45-day renewal workflow pilot. +- Partner channel begins after SOC 2 Type II readiness. +- Target 120 paid teams by month 18. + +## Ask + +Raise $2.4M to fund 18 months of product, security, and go-to-market execution. + +## Proof Points + +- Renewal-prep packets completed 38% faster in prototype testing. +- Missed follow-up rate dropped from 14% to 5% across sampled accounts. +- Eight pilot teams asked to keep the prototype after the trial. diff --git a/packages/opencode/script/office-route-eval/fixtures/sales-2026.csv b/packages/opencode/script/office-route-eval/fixtures/sales-2026.csv new file mode 100644 index 000000000..b1944dfac --- /dev/null +++ b/packages/opencode/script/office-route-eval/fixtures/sales-2026.csv @@ -0,0 +1,19 @@ +month,region,segment,channel,revenue,gross_margin,customers +2026-01,North,Enterprise,Direct,240000,156000,18 +2026-01,South,Midmarket,Partner,185000,105450,24 +2026-01,West,SMB,Self-serve,92000,51520,41 +2026-02,North,Enterprise,Direct,265000,174900,19 +2026-02,South,Midmarket,Partner,194000,112520,25 +2026-02,West,SMB,Self-serve,99000,57420,44 +2026-03,North,Enterprise,Direct,278000,183480,20 +2026-03,South,Midmarket,Partner,205000,120950,27 +2026-03,West,SMB,Self-serve,111000,65490,49 +2026-04,North,Enterprise,Direct,301000,204680,22 +2026-04,South,Midmarket,Partner,213000,127800,28 +2026-04,West,SMB,Self-serve,119000,71400,53 +2026-05,North,Enterprise,Direct,326000,224940,23 +2026-05,South,Midmarket,Partner,229000,139690,31 +2026-05,West,SMB,Self-serve,128000,78080,58 +2026-06,North,Enterprise,Direct,349000,244300,25 +2026-06,South,Midmarket,Partner,241000,149420,33 +2026-06,West,SMB,Self-serve,136000,85680,62 diff --git a/packages/opencode/script/office-route-eval/report.md b/packages/opencode/script/office-route-eval/report.md new file mode 100644 index 000000000..d49a61a10 --- /dev/null +++ b/packages/opencode/script/office-route-eval/report.md @@ -0,0 +1,43 @@ +# Office Route Eval Report + +## Verdict + +Do not open a formal OfficeCLI replacement PR yet. Python route beats OfficeCLI on xlsx and ties docx, but pptx still fails 0/3 on explicit font-size XML. + +Next smallest boundary: keep this eval harness as the baseline, then harden only the Python pptx skill/template so generated decks set run-level font sizes and pass the existing pptx judge in at least 2/3 rounds. + +## Aggregate + +| Task | Route | Passes | Median Seconds | Median Commands | +|---|---:|---:|---:|---:| +| xlsx-dashboard | officecli | 0/3 | 265 | 33 | +| xlsx-dashboard | python | 3/3 | 72 | 3 | +| docx-board-memo | officecli | 3/3 | 123 | 22 | +| docx-board-memo | python | 3/3 | 84 | 5 | +| pptx-pitch-deck | officecli | 0/3 | 296 | 29 | +| pptx-pitch-deck | python | 0/3 | 94 | 4 | + +## Runs + +| Task | Route | Round | Result | Score | Seconds | Commands | Failures | +|---|---:|---:|---|---:|---:|---:|---| +| docx-board-memo | officecli | 1 | pass | 100 | 123 | 22 | | +| docx-board-memo | python | 1 | pass | 100 | 84 | 6 | | +| pptx-pitch-deck | officecli | 1 | fail | 82 | 387 | 40 | Target artifact does not exist at required path; found at ./orion-assist-pitch.pptx. | +| pptx-pitch-deck | python | 1 | fail | 82 | 94 | 4 | PPTX has no explicit run font sizes. | +| xlsx-dashboard | officecli | 1 | fail | 64 | 265 | 33 | XLSX has no chart XML.; XLSX has fewer than three formulas. | +| xlsx-dashboard | python | 1 | pass | 100 | 72 | 3 | | +| docx-board-memo | officecli | 2 | pass | 100 | 134 | 16 | | +| docx-board-memo | python | 2 | pass | 100 | 61 | 3 | | +| pptx-pitch-deck | officecli | 2 | fail | 64 | 296 | 29 | PPTX has no chart XML.; PPTX has no explicit run font sizes. | +| pptx-pitch-deck | python | 2 | fail | 82 | 76 | 4 | PPTX has no explicit run font sizes. | +| xlsx-dashboard | officecli | 2 | fail | 46 | 117 | 23 | XLSX has no chart XML.; XLSX has fewer than three formulas.; XLSX contains formula error token #REF!. | +| xlsx-dashboard | python | 2 | pass | 100 | 111 | 4 | | +| docx-board-memo | officecli | 3 | pass | 100 | 113 | 52 | | +| docx-board-memo | python | 3 | pass | 100 | 90 | 5 | | +| pptx-pitch-deck | officecli | 3 | fail | 64 | 129 | 18 | PPTX has no chart XML.; PPTX has no explicit run font sizes. | +| pptx-pitch-deck | python | 3 | fail | 82 | 114 | 6 | PPTX has no explicit run font sizes. | +| xlsx-dashboard | officecli | 3 | fail | 82 | 448 | 37 | OfficeCLI route appears to create the final artifact through Python Office libraries. | +| xlsx-dashboard | python | 3 | pass | 100 | 66 | 3 | | + +Replacement PR bar: Python route must pass all three task families in at least two of three rounds with zero route-policy failures. \ No newline at end of file diff --git a/packages/opencode/script/office-route-eval/route-skills/officecli-eval-policy/SKILL.md b/packages/opencode/script/office-route-eval/route-skills/officecli-eval-policy/SKILL.md new file mode 100644 index 000000000..29d3342d7 --- /dev/null +++ b/packages/opencode/script/office-route-eval/route-skills/officecli-eval-policy/SKILL.md @@ -0,0 +1,22 @@ +--- +name: officecli-eval-policy +description: Use for Office route eval tasks. It defines the route boundary for #1273 OfficeCLI replacement evaluation. +--- + +# OfficeCLI Eval Policy + +You are running the current PawWork OfficeCLI route for an eval. + +Hard route rules: + +- Use `officecli` for the final `.xlsx`, `.docx`, or `.pptx` artifact. +- Do not use LibreOffice or aliases: `libreoffice`, `soffice`, `lowriter`, `localc`, `loffice`. +- Do not use `uv`. +- Do not create the final Office artifact with Python Office libraries such as `openpyxl`, `python-docx`, or `python-pptx`. +- Lightweight text, CSV, JSON, or shell preprocessing is allowed if the final Office file is created or edited through `officecli`. + +Delivery rules: + +- Write the requested artifact under `./artifacts/`. +- Write `./artifacts/artifact-summary.json` after the artifact exists. +- Run the strongest available non-LibreOffice validation before declaring success, such as `officecli validate` or `officecli view`. diff --git a/packages/opencode/script/office-route-eval/route-skills/python-office-eval/SKILL.md b/packages/opencode/script/office-route-eval/route-skills/python-office-eval/SKILL.md new file mode 100644 index 000000000..b90d3452d --- /dev/null +++ b/packages/opencode/script/office-route-eval/route-skills/python-office-eval/SKILL.md @@ -0,0 +1,28 @@ +--- +name: python-office-eval +description: Use for Python + uv Office eval tasks. It defines the replacement-route boundary for #1273. +--- + +# Python Office Eval + +You are running the Python + uv + skills route for an Office artifact eval. + +Hard route rules: + +- Use `uv` and Python to create the requested `.xlsx`, `.docx`, or `.pptx`. +- Do not call `officecli`. +- Do not use LibreOffice or aliases: `libreoffice`, `soffice`, `lowriter`, `localc`, `loffice`. +- Use the provided `pyproject.toml` in the working directory. It includes `openpyxl`, `python-docx`, and `python-pptx`. + +Preferred build pattern: + +1. Write a small Python script in the working directory. +2. Run it with `uv run python