From bafa22080db3315a43377cb66c6dee12bda97f93 Mon Sep 17 00:00:00 2001 From: DevFlex-AI Date: Fri, 31 Jul 2026 17:36:21 +0000 Subject: [PATCH 1/2] feat(opencode): add eval command for agent benchmark harness --- packages/opencode/src/cli/cmd/eval.ts | 273 ++++++++++++++++++ packages/opencode/src/cli/cmd/eval/case.ts | 126 ++++++++ packages/opencode/src/index.ts | 2 + .../test/cli/eval/eval-process.test.ts | 68 +++++ 4 files changed, 469 insertions(+) create mode 100644 packages/opencode/src/cli/cmd/eval.ts create mode 100644 packages/opencode/src/cli/cmd/eval/case.ts create mode 100644 packages/opencode/test/cli/eval/eval-process.test.ts diff --git a/packages/opencode/src/cli/cmd/eval.ts b/packages/opencode/src/cli/cmd/eval.ts new file mode 100644 index 000000000000..1b7809b68f09 --- /dev/null +++ b/packages/opencode/src/cli/cmd/eval.ts @@ -0,0 +1,273 @@ +// CLI entry point for `bolt eval`: the agent evaluation benchmark harness. +// +// Each eval case is a markdown file (frontmatter + prompt body, see +// ./eval/case.ts). The harness runs every case against an in-process server in +// its own isolated temp workspace, waits for the session to finish, grades the +// workspace with the case's checks, and reports pass/fail plus cost and token +// usage. Exit code is 1 when any case fails. +import type { Argv } from "yargs" +import path from "path" +import os from "os" +import { EOL } from "os" +import { mkdtemp, rm } from "node:fs/promises" +import { Effect } from "effect" +import { createOpencodeClient, type OpencodeClient } from "@opencode-ai/sdk/v2" +import { UI } from "../ui" +import { effectCmd, fail } from "../effect-cmd" +import { discover, load, runCheck, describeCheck, type CheckResult, type Info } from "./eval/case" + +type ModelInput = Parameters[0]["model"] + +function pick(value: string | undefined): ModelInput | undefined { + if (!value) return undefined + const [providerID, ...rest] = value.split("/") + return { + providerID, + modelID: rest.join("/"), + } as ModelInput +} + +interface CaseResult { + item: Info + file: string + workspace: string + sessionID?: string + passed: boolean + checks: CheckResult[] + errors: string[] + durationMs: number + cost?: number + tokens?: unknown +} + +export const EvalCommand = effectCmd({ + command: "eval [paths..]", + describe: "run agent evaluation cases and grade the results", + // Each case loads its own instance for its temp workspace through the + // server's per-directory routing; no instance is needed for the cwd. + instance: false, + builder: (yargs: Argv) => + yargs + .positional("paths", { + describe: "eval case files or directories (default: ./evals)", + type: "string", + array: true, + default: ["evals"], + }) + .option("model", { + type: "string", + alias: ["m"], + describe: "default model in provider/model format (cases can override)", + }) + .option("agent", { + type: "string", + describe: "default agent (cases can override)", + }) + .option("format", { + type: "string", + choices: ["default", "json"], + default: "default", + describe: "format: default (formatted) or json (one JSON object per line)", + }) + .option("timeout", { + type: "number", + default: 300, + describe: "per-case timeout in seconds (cases can override)", + }) + .option("keep", { + type: "boolean", + default: false, + describe: "keep case workspaces on disk for debugging", + }), + handler: Effect.fn("Cli.eval")(function* (args) { + const { ServerAuth } = yield* Effect.promise(() => import("@/server/auth")) + const discovered = yield* Effect.promise(() => discover(args.paths)) + if (discovered.missing.length) { + return yield* fail(`No such file or directory: ${discovered.missing.join(", ")}`) + } + if (discovered.files.length === 0) { + return yield* fail(`No eval cases found in: ${args.paths.join(", ")}`) + } + const cases = yield* Effect.promise(() => Promise.all(discovered.files.map((file) => load(file)))) + + yield* Effect.promise(async () => { + const json = args.format === "json" + const fetchFn = (async (input: RequestInfo | URL, init?: RequestInit) => { + const { Server } = await import("@/server/server") + const request = new Request(input, init) + const headers = new Headers(request.headers) + const auth = ServerAuth.header() + if (auth) headers.set("Authorization", auth) + return Server.Default().app.fetch(new Request(request, { headers })) + }) as typeof globalThis.fetch + + function emit(type: string, data: Record) { + if (!json) return + process.stdout.write(JSON.stringify({ type, timestamp: Date.now(), ...data }) + EOL) + } + + async function execute(item: Info, file: string): Promise { + const start = Date.now() + const workspace = await mkdtemp(path.join(os.tmpdir(), "bolt-eval-")) + const errors: string[] = [] + const failure = (message: string): CaseResult => ({ + item, + file, + workspace, + passed: false, + checks: [], + errors: [...errors, message], + durationMs: Date.now() - start, + }) + + for (const [relative, content] of Object.entries(item.files ?? {})) { + await Bun.write(path.join(workspace, relative), content) + } + + const sdk = createOpencodeClient({ + baseUrl: "http://opencode.internal", + fetch: fetchFn, + directory: workspace, + }) + const session = await sdk.session.create({ + title: `eval: ${item.name}`, + permission: [ + { permission: "question", pattern: "*", action: "deny" }, + { permission: "plan_enter", pattern: "*", action: "deny" }, + { permission: "plan_exit", pattern: "*", action: "deny" }, + ], + }) + const sessionID = session.data?.id + if (!sessionID) return failure("failed to create session") + + // The workspace is an isolated throwaway temp directory, so tool + // permissions are auto-approved; question/plan permissions stay + // denied via the session ruleset above. + const events = await sdk.event.subscribe() + const pump = (async () => { + for await (const event of events.stream) { + if (event.type === "permission.asked") { + const permission = event.properties + if (permission.sessionID !== sessionID) continue + await sdk.permission.reply({ requestID: permission.id, reply: "once" }) + } + if (event.type === "session.error") { + const props = event.properties + if (props.sessionID !== sessionID || !props.error) continue + const err = + "data" in props.error && props.error.data && "message" in props.error.data + ? String(props.error.data.message) + : String(props.error.name) + errors.push(err) + } + if ( + event.type === "session.status" && + event.properties.sessionID === sessionID && + event.properties.status.type === "idle" + ) { + break + } + } + })().catch(() => {}) + + let timedOut = false + const timeoutMs = (item.timeout ?? args.timeout) * 1000 + const timer = setTimeout(() => { + timedOut = true + void sdk.session.abort({ sessionID }).catch(() => {}) + }, timeoutMs) + const result = await sdk.session + .prompt({ + sessionID, + model: pick(item.model ?? args.model), + agent: item.agent ?? args.agent, + parts: [{ type: "text", text: item.prompt }], + }) + .finally(() => clearTimeout(timer)) + if (result.error) { + errors.push(JSON.stringify(result.error)) + } else { + await pump + } + if (timedOut) errors.push(`timed out after ${timeoutMs / 1000}s`) + + const checks: CheckResult[] = [] + for (const check of item.expect) { + checks.push(await runCheck(check, workspace)) + } + const info = await sdk.session + .get({ sessionID }) + .then((response) => response.data) + .catch(() => undefined) + return { + item, + file, + workspace, + sessionID, + passed: errors.length === 0 && checks.every((check) => check.passed), + checks, + errors, + durationMs: Date.now() - start, + cost: info?.cost, + tokens: info?.tokens, + } + } + + function report(result: CaseResult) { + emit("eval_case", { + name: result.item.name, + file: result.file, + sessionID: result.sessionID, + passed: result.passed, + checks: result.checks.map((entry) => ({ + ...entry.check, + passed: entry.passed, + detail: entry.detail, + })), + errors: result.errors, + durationMs: result.durationMs, + cost: result.cost, + tokens: result.tokens, + workspace: args.keep ? result.workspace : undefined, + }) + if (json) return + const seconds = `${(result.durationMs / 1000).toFixed(1)}s` + const cost = result.cost ? ` · $${result.cost.toFixed(4)}` : "" + const marker = result.passed ? UI.Style.TEXT_SUCCESS + "✓" : UI.Style.TEXT_DANGER_BOLD + "✗" + UI.println( + `${marker} ${UI.Style.TEXT_NORMAL}${result.item.name} ${UI.Style.TEXT_DIM}${seconds}${cost}${UI.Style.TEXT_NORMAL}`, + ) + result.checks + .filter((entry) => !entry.passed) + .forEach((entry) => + UI.println( + ` ${UI.Style.TEXT_DANGER_BOLD}✗${UI.Style.TEXT_NORMAL} ${describeCheck(entry.check)}${UI.Style.TEXT_DIM}${entry.detail ? ` — ${entry.detail}` : ""}${UI.Style.TEXT_NORMAL}`, + ), + ) + result.errors.forEach((error) => UI.println(` ${UI.Style.TEXT_DANGER_BOLD}!${UI.Style.TEXT_NORMAL} ${error}`)) + if (args.keep) UI.println(` ${UI.Style.TEXT_DIM}workspace: ${result.workspace}${UI.Style.TEXT_NORMAL}`) + } + + const results: CaseResult[] = [] + for (const [index, item] of cases.entries()) { + const result = await execute(item, discovered.files[index]) + if (!args.keep) await rm(result.workspace, { recursive: true, force: true }).catch(() => {}) + report(result) + results.push(result) + } + + const passed = results.filter((result) => result.passed).length + const cost = results.reduce((sum, result) => sum + (result.cost ?? 0), 0) + const durationMs = results.reduce((sum, result) => sum + result.durationMs, 0) + emit("eval_summary", { total: results.length, passed, failed: results.length - passed, durationMs, cost }) + if (!json) { + UI.empty() + const style = passed === results.length ? UI.Style.TEXT_SUCCESS : UI.Style.TEXT_DANGER_BOLD + UI.println( + `${style}${passed}/${results.length} passed${UI.Style.TEXT_NORMAL} ${UI.Style.TEXT_DIM}${(durationMs / 1000).toFixed(1)}s${cost ? ` · $${cost.toFixed(4)}` : ""}${UI.Style.TEXT_NORMAL}`, + ) + } + if (passed !== results.length) process.exitCode = 1 + }) + }), +}) diff --git a/packages/opencode/src/cli/cmd/eval/case.ts b/packages/opencode/src/cli/cmd/eval/case.ts new file mode 100644 index 000000000000..597990c91be7 --- /dev/null +++ b/packages/opencode/src/cli/cmd/eval/case.ts @@ -0,0 +1,126 @@ +import path from "path" +import { Schema } from "effect" +import { Glob } from "@opencode-ai/core/util/glob" +import { Filesystem } from "@/util/filesystem" +import { ConfigMarkdown } from "@/config/markdown" +import { ConfigParse } from "@/config/parse" + +// Eval cases are markdown files with YAML frontmatter: the frontmatter carries +// the model/agent overrides and the grading checks, the body is the prompt +// handed to the agent. Mirrors the agent/command config file convention. + +export const Check = Schema.Union([ + Schema.Struct({ + type: Schema.Literal("file_exists"), + path: Schema.String, + }), + Schema.Struct({ + type: Schema.Literal("file_contains"), + path: Schema.String, + text: Schema.optional(Schema.String), + pattern: Schema.optional(Schema.String), + }), + Schema.Struct({ + type: Schema.Literal("command"), + run: Schema.String, + timeout: Schema.optional(Schema.Number), + }), +]) +export type Check = typeof Check.Type + +export const Info = Schema.Struct({ + name: Schema.String, + description: Schema.optional(Schema.String), + model: Schema.optional(Schema.String), + agent: Schema.optional(Schema.String), + timeout: Schema.optional(Schema.Number), + files: Schema.optional(Schema.Record(Schema.String, Schema.String)), + expect: Schema.mutable(Schema.Array(Check)), + prompt: Schema.String, +}) +export type Info = typeof Info.Type + +// Expands targets into a sorted, deduplicated list of case files: directories +// are scanned recursively for *.md, files are taken as-is. Unknown paths are +// returned separately so the caller can fail with a user-visible message. +export async function discover(targets: string[]) { + const found: string[] = [] + const missing: string[] = [] + for (const target of targets) { + const resolved = path.resolve(target) + const stat = Filesystem.stat(resolved) + if (!stat) { + missing.push(target) + continue + } + if (stat.isDirectory()) { + found.push(...(await Glob.scan("**/*.md", { cwd: resolved, absolute: true, dot: true, symlink: true }))) + continue + } + found.push(resolved) + } + return { files: [...new Set(found)].sort(), missing } +} + +export async function load(file: string): Promise { + const md = await ConfigMarkdown.parse(file) + const config = { + name: path.basename(file, ".md"), + ...md.data, + prompt: md.content.trim(), + } + return ConfigParse.schema(Info, config, file) +} + +export interface CheckResult { + check: Check + passed: boolean + detail?: string +} + +export function describeCheck(check: Check) { + if (check.type === "file_exists") return `file_exists ${check.path}` + if (check.type === "file_contains") + return `file_contains ${check.path} · ${check.pattern ?? check.text ?? "(no matcher)"}` + return `command ${check.run}` +} + +export async function runCheck(check: Check, workspace: string): Promise { + if (check.type === "file_exists") { + const passed = await Filesystem.exists(path.join(workspace, check.path)) + return { check, passed, detail: passed ? undefined : "file not found" } + } + + if (check.type === "file_contains") { + if (check.text === undefined && check.pattern === undefined) { + return { check, passed: false, detail: "file_contains requires text or pattern" } + } + const target = path.join(workspace, check.path) + if (!(await Filesystem.exists(target))) { + return { check, passed: false, detail: "file not found" } + } + const content = await Filesystem.readText(target) + const passed = + check.pattern !== undefined + ? new RegExp(check.pattern, "m").test(content) + : check.text !== undefined && content.includes(check.text) + return { check, passed, detail: passed ? undefined : "no match" } + } + + const proc = Bun.spawn(["bash", "-c", check.run], { + cwd: workspace, + stdout: "pipe", + stderr: "pipe", + timeout: (check.timeout ?? 120) * 1000, + }) + const stdout = await new Response(proc.stdout).text() + const stderr = await new Response(proc.stderr).text() + const exitCode = await proc.exited + if (exitCode === 0) return { check, passed: true } + const output = (stdout + stderr).trim() + return { + check, + passed: false, + detail: `exit ${exitCode}${output ? `: ${output.slice(0, 400)}` : ""}`, + } +} diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index ed3787b0bd0f..8bef913948b8 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -14,6 +14,7 @@ import { FormatError } from "./cli/error" import { ServeCommand } from "./cli/cmd/serve" import { DebugCommand } from "./cli/cmd/debug" import { StatsCommand } from "./cli/cmd/stats" +import { EvalCommand } from "./cli/cmd/eval" import { LogsCommand } from "./cli/cmd/logs" import { McpCommand } from "./cli/cmd/mcp" import { GithubCommand } from "./cli/cmd/github" @@ -95,6 +96,7 @@ const cli = yargs(args) .command(WebCommand) .command(ModelsCommand) .command(StatsCommand) + .command(EvalCommand) .command(LogsCommand) .command(ExportCommand) .command(ImportCommand) diff --git a/packages/opencode/test/cli/eval/eval-process.test.ts b/packages/opencode/test/cli/eval/eval-process.test.ts new file mode 100644 index 000000000000..1051fc672a00 --- /dev/null +++ b/packages/opencode/test/cli/eval/eval-process.test.ts @@ -0,0 +1,68 @@ +// End-to-end tests for `bolt eval`: spawn the real CLI against the mock LLM, +// grade a passing and a failing case, and assert the JSON report + exit code. +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import path from "node:path" +import { cliIt, testModelID } from "../../lib/cli-process" + +const passCase = `--- +expect: + - type: file_contains + path: out.txt + text: hello +--- +Create out.txt containing hello. +` + +const failCase = `--- +expect: + - type: file_exists + path: missing.txt +--- +Do nothing. +` + +describe("bolt eval", () => { + cliIt.concurrent( + "grades cases and reports results as json", + ({ llm, home, opencode }) => + Effect.gen(function* () { + yield* Effect.promise(() => Bun.write(path.join(home, "evals/a-fail.md"), failCase)) + yield* Effect.promise(() => Bun.write(path.join(home, "evals/b-pass.md"), passCase)) + + // Cases run in sorted file order: a-fail first (one text turn), then + // b-pass (bash tool call writing out.txt, then a closing text turn). + yield* llm.text("done") + yield* llm.tool("bash", { command: "echo hello > out.txt", description: "Write out.txt" }) + yield* llm.text("done") + + const result = yield* opencode.spawn(["eval", "evals", "--model", testModelID, "--format", "json"], { + timeoutMs: 120_000, + }) + opencode.expectExit(result, 1, "eval") + + const events = opencode.parseJsonEvents(result.stdout) + const cases = events.filter((event) => event.type === "eval_case") + expect(cases).toHaveLength(2) + + const failed = cases.find((event) => event.name === "a-fail") + expect(failed?.passed).toBe(false) + const passedCase = cases.find((event) => event.name === "b-pass") + expect(passedCase?.passed).toBe(true) + + const summary = events.find((event) => event.type === "eval_summary") + expect(summary?.total).toBe(2) + expect(summary?.passed).toBe(1) + expect(summary?.failed).toBe(1) + }), + 240_000, + ) + + cliIt.concurrent("fails when no cases are found", ({ opencode }) => + Effect.gen(function* () { + const result = yield* opencode.spawn(["eval", "does-not-exist"], { timeoutMs: 60_000 }) + opencode.expectExit(result, 1, "eval-missing") + expect(result.stderr).toContain("does-not-exist") + }), + ) +}) From 2ecacd6293772809b694dbcfdb6e7445358f736a Mon Sep 17 00:00:00 2001 From: DevFlex-AI Date: Fri, 31 Jul 2026 17:48:48 +0000 Subject: [PATCH 2/2] feat(opencode): print eval case progress before each run --- packages/opencode/src/cli/cmd/eval.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/opencode/src/cli/cmd/eval.ts b/packages/opencode/src/cli/cmd/eval.ts index 1b7809b68f09..f890ac8bd49e 100644 --- a/packages/opencode/src/cli/cmd/eval.ts +++ b/packages/opencode/src/cli/cmd/eval.ts @@ -250,6 +250,10 @@ export const EvalCommand = effectCmd({ const results: CaseResult[] = [] for (const [index, item] of cases.entries()) { + if (!json) { + const model = item.model ?? args.model + UI.println(`${UI.Style.TEXT_DIM}● ${item.name} running${model ? ` on ${model}` : ""}…${UI.Style.TEXT_NORMAL}`) + } const result = await execute(item, discovered.files[index]) if (!args.keep) await rm(result.workspace, { recursive: true, force: true }).catch(() => {}) report(result)