diff --git a/Cargo.lock b/Cargo.lock index 9f014e3bd7d0..b5af4a3fe841 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1169,7 +1169,7 @@ dependencies = [ "quote", "regex", "rustc-hash 1.1.0", - "shlex", + "shlex 1.3.0", "syn 2.0.117", ] diff --git a/evals/open-model-gym/Justfile b/evals/open-model-gym/Justfile index 6b08e5169b61..09ccdf62d3bf 100644 --- a/evals/open-model-gym/Justfile +++ b/evals/open-model-gym/Justfile @@ -7,6 +7,14 @@ default: run run: _install cd suite && npm run test +# Full run with artifacts isolated under ~/.goose/gym-runs/ (keeps the repo clean) +run-clean: _install + #!/usr/bin/env bash + set -euo pipefail + export GYM_OUTPUT_DIR="$HOME/.goose/gym-runs/$(date +%Y%d%m%H%M%S)" + echo "Artifacts ā†’ $GYM_OUTPUT_DIR" + cd suite && npm run test + # Quick test - file-editing + everyday-app-automation, single run each (no repetition) test: _install cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1 @@ -19,9 +27,12 @@ scenario name: _install agent name: _install cd suite && npx tsx src/runner.ts --agent={{name}} -# Open report in browser +# Open report in browser (honors GYM_OUTPUT_DIR if set) report: - open report.html + #!/usr/bin/env bash + set -euo pipefail + dir="${GYM_OUTPUT_DIR:-.}" + open "${dir/#\~/$HOME}/report.html" # Install all dependencies install: diff --git a/evals/open-model-gym/README.md b/evals/open-model-gym/README.md index fb5e39197f19..7aa8899a7f2b 100644 --- a/evals/open-model-gym/README.md +++ b/evals/open-model-gym/README.md @@ -267,6 +267,7 @@ Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log` | Command | Description | |---------|-------------| | `just run` | Full test run (3 reps each, worst kept) | +| `just run-clean` | Full run with artifacts isolated under `~/.goose/gym-runs/` | | `just test` | Quick run (1 rep each) | | `just scenario ` | Run specific scenario | | `just agent ` | Run specific agent | @@ -284,6 +285,9 @@ npx tsx src/runner.ts --run-count=5 # Don't auto-open browser npx tsx src/runner.ts --no-open +# Redirect all run artifacts outside the repo (see Output below) +npx tsx src/runner.ts --output-dir=~/.goose/gym-runs/latest + # Raise the per-agent timeout (seconds) for slow local models on heavy # scenarios. Default 300s; also settable via GYM_AGENT_TIMEOUT. npx tsx src/runner.ts --agent-timeout=1200 @@ -293,3 +297,27 @@ npx tsx src/runner.ts --agent-timeout=1200 - `report.html` ā€” Live-updating HTML matrix showing pass/fail status, duration, and validation details - `logs/` ā€” Full agent output logs for each run + +By default these (plus the cache, scratch workdir, and isolated agent config +roots `.goose-root/` / `.opencode-root/` / `.pi-root/`) are written inside the +gym directory. They're gitignored, but still pile up in your checkout ā€” awkward +if you want to run the bench regularly or from a worktree. + +To keep the repo clean, redirect **all** run artifacts to a single base +directory with the `GYM_OUTPUT_DIR` env var (or the `--output-dir=` flag). +`config.yaml` and `scenarios/` are still read from the repo. + +```bash +# Everything lands under a timestamped dir outside the repo (YYYYDDMMHHMMSS) +GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run + +# Convenience recipe that does the timestamping for you +just run-clean + +# View the report from a redirected run +GYM_OUTPUT_DIR=~/.goose/gym-runs/20261406101500 just report +``` + +> Note: the run cache lives under the output dir too, so a fresh timestamped +> dir means a fresh (cold) cache. Point `GYM_OUTPUT_DIR` at a stable directory +> if you want cache reuse across runs. diff --git a/evals/open-model-gym/suite/src/runner.ts b/evals/open-model-gym/suite/src/runner.ts index 124ff08709bc..62ebff6f9b5d 100644 --- a/evals/open-model-gym/suite/src/runner.ts +++ b/evals/open-model-gym/suite/src/runner.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import { mkdirSync, writeFileSync, rmSync, readdirSync, existsSync, copyFileSync } from "node:fs"; -import { join, basename, dirname } from "node:path"; +import { join, basename, dirname, resolve } from "node:path"; import { homedir } from "node:os"; -import { execSync } from "node:child_process"; +import { execSync, execFileSync } from "node:child_process"; import { parse, stringify } from "yaml"; import { readFileSync } from "node:fs"; import { createHash } from "node:crypto"; @@ -89,6 +89,41 @@ interface CacheIndex { } // ============================================================================= +// Output directory resolution +// ============================================================================= +// All run artifacts (cache, isolated agent config roots, scratch workdir, logs, +// and the HTML report) live under a single base directory. By default this is +// the in-repo gym directory, so existing behavior is unchanged. Set the +// GYM_OUTPUT_DIR env var or pass --output-dir= to redirect everything +// outside the repo and keep your checkout clean, e.g.: +// +// GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run +// +// config.yaml and scenarios/ are inputs and always read from the repo. + +const SUITE_DIR = join(import.meta.dirname, ".."); // .../open-model-gym/suite +const GYM_DIR = join(import.meta.dirname, "../.."); // .../open-model-gym + +function expandHome(p: string): string { + return p === "~" || p.startsWith("~/") ? join(homedir(), p.slice(1)) : p; +} + +// Resolved output base, or null to fall back to the legacy in-repo locations. +const OUTPUT_DIR: string | null = (() => { + const flag = process.argv + .find((a) => a.startsWith("--output-dir=")) + ?.split("=")[1]; + const base = flag ?? process.env.GYM_OUTPUT_DIR; + // Resolve to an absolute path: runners exec with cwd set to the workdir, so a + // relative base would make prompt/log paths resolve against the wrong dir. + return base ? resolve(expandHome(base)) : null; +})(); + +// Resolve an artifact path under OUTPUT_DIR when set, else its legacy anchor. +function artifactPath(name: string, legacyAnchor: string): string { + return join(OUTPUT_DIR ?? legacyAnchor, name); +} + // Agent timeout // ============================================================================= // Per-invocation timeout for an agent run, in milliseconds. Larger local models @@ -107,7 +142,7 @@ const AGENT_TIMEOUT_MS = (() => { // Cache Utilities // ============================================================================= -const CACHE_DIR = join(import.meta.dirname, "../.cache"); +const CACHE_DIR = artifactPath(".cache", SUITE_DIR); const CACHE_INDEX_PATH = join(CACHE_DIR, "index.json"); const CACHE_LOGS_DIR = join(CACHE_DIR, "logs"); const CACHE_VERSION = 1; @@ -317,7 +352,7 @@ const PLATFORM_EXTENSIONS = new Set([ ]); // Isolated goose config directory -const GOOSE_ROOT = join(import.meta.dirname, "../.goose-root"); +const GOOSE_ROOT = artifactPath(".goose-root", SUITE_DIR); const GOOSE_CONFIG_DIR = join(GOOSE_ROOT, "config"); function generateGooseConfig(model: ModelConfig, runner: RunnerConfig): object { @@ -418,7 +453,7 @@ async function runGooseAgent( // ============================================================================= // Isolated opencode config directory -const OPENCODE_ROOT = join(import.meta.dirname, "../.opencode-root"); +const OPENCODE_ROOT = artifactPath(".opencode-root", SUITE_DIR); function generateOpenCodeConfig(model: ModelConfig, runner: RunnerConfig, workdir: string): object { const mcp: Record = {}; @@ -517,7 +552,7 @@ async function runOpenCodeAgent( // MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter` // Isolated Pi config directory (like Goose/OpenCode) -const PI_CONFIG_DIR = join(import.meta.dirname, "../.pi-root"); +const PI_CONFIG_DIR = artifactPath(".pi-root", SUITE_DIR); // User's real Pi config (for copying auth.json) const PI_USER_CONFIG = join(homedir(), ".pi", "agent"); @@ -1000,11 +1035,15 @@ function generateHtmlReport( ): void { const { isRunning = false, allPairs = [] } = options; - // Read and embed gym.png as base64 - const rootDir = join(outputPath, ".."); + // Read and embed gym.png as base64. Prefer one sitting next to the report + // (legacy in-repo layout); otherwise fall back to the copy in the source tree + // so the image still embeds when output is redirected via GYM_OUTPUT_DIR. let gymBase64 = ""; try { - const gymPath = join(rootDir, "gym.png"); + const adjacent = join(outputPath, "..", "gym.png"); + const gymPath = existsSync(adjacent) + ? adjacent + : join(import.meta.dirname, "gym.png"); gymBase64 = readFileSync(gymPath).toString("base64"); } catch (e) { // gym.png not found, will use external reference @@ -1344,6 +1383,7 @@ function generateHtmlReport( `; + mkdirSync(dirname(outputPath), { recursive: true }); writeFileSync(outputPath, html); console.log(`\nšŸ“Š Report saved to: ${outputPath}`); } @@ -1382,12 +1422,11 @@ async function main() { return; } - const rootDir = join(import.meta.dirname, "../.."); - const configPath = join(rootDir, "config.yaml"); + const configPath = join(GYM_DIR, "config.yaml"); const scenariosDir = join(import.meta.dirname, "../scenarios"); - const workdir = join(import.meta.dirname, "../.workdir"); - const logsDir = join(rootDir, "logs"); - const reportPath = join(rootDir, "report.html"); + const workdir = artifactPath(".workdir", SUITE_DIR); + const logsDir = artifactPath("logs", GYM_DIR); + const reportPath = artifactPath("report.html", GYM_DIR); const config = loadConfig(configPath); let scenarios = loadAllScenarios(scenariosDir); @@ -1444,6 +1483,7 @@ async function main() { } const mcpHarnessHash = getMcpHarnessHash(); + console.log(`Output: ${OUTPUT_DIR ?? GYM_DIR}${OUTPUT_DIR ? "" : " (in-repo; set GYM_OUTPUT_DIR to redirect)"}`); console.log(`Models: ${config.models.map((m) => m.name).join(", ")}`); console.log(`Runners: ${config.runners.map((r) => r.name).join(", ")}`); console.log(`Running ${pairs.length} test pairs (${RUN_COUNT}x each, worst result kept)`); @@ -1476,7 +1516,7 @@ async function main() { if (!browserOpened) { generateHtmlReport(results, reportPath, { isRunning: true, allPairs: pairs }); if (!noOpen) { - execSync(`open "${reportPath}"`); + execFileSync("open", [reportPath]); } browserOpened = true; } @@ -1511,10 +1551,10 @@ async function main() { } generateHtmlReport(results, reportPath, { isRunning: false, allPairs: pairs }); - + // If everything was cached, open browser now with final report if (!browserOpened && !noOpen) { - execSync(`open "${reportPath}"`); + execFileSync("open", [reportPath]); } printResults(results);