From 7d86b89ec338b5f1bceb6b1178b5a18d49402df8 Mon Sep 17 00:00:00 2001 From: Kyle De Freitas Date: Sun, 14 Jun 2026 18:00:31 -0400 Subject: [PATCH] feat(gym): make Open Model Gym output dir configurable All run artifacts (cache, isolated agent config roots, scratch workdir, logs, and report.html) were hardcoded relative to the suite source dir, so every run wrote into the checkout. They're gitignored, but still pile up in the repo and duplicate per worktree, making it awkward to run the bench regularly. Add a single base-dir knob: set GYM_OUTPUT_DIR (or pass --output-dir=) to redirect every artifact outside the repo. Defaults to the existing in-repo locations, so behavior is unchanged when unset. config.yaml and scenarios/ remain inputs read from the repo. - runner.ts: centralize path resolution via OUTPUT_DIR + artifactPath(); resolve gym.png from the source tree when redirected; mkdir the report parent dir; print the resolved output location in the run banner. - Justfile: add `run-clean` (isolates under ~/.goose/gym-runs/); `report` honors GYM_OUTPUT_DIR. - README: document GYM_OUTPUT_DIR/--output-dir and the cache-reuse caveat. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Kyle De Freitas --- evals/open-model-gym/Justfile | 15 ++++- evals/open-model-gym/README.md | 28 +++++++++ evals/open-model-gym/suite/src/runner.ts | 75 ++++++++++++++++++------ 3 files changed, 99 insertions(+), 19 deletions(-) diff --git a/evals/open-model-gym/Justfile b/evals/open-model-gym/Justfile index 6b08e5169b61..09ccdf62d3bf 100644 --- a/evals/open-model-gym/Justfile +++ b/evals/open-model-gym/Justfile @@ -7,6 +7,14 @@ default: run run: _install cd suite && npm run test +# Full run with artifacts isolated under ~/.goose/gym-runs/ (keeps the repo clean) +run-clean: _install + #!/usr/bin/env bash + set -euo pipefail + export GYM_OUTPUT_DIR="$HOME/.goose/gym-runs/$(date +%Y%d%m%H%M%S)" + echo "Artifacts ā†’ $GYM_OUTPUT_DIR" + cd suite && npm run test + # Quick test - file-editing + everyday-app-automation, single run each (no repetition) test: _install cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1 @@ -19,9 +27,12 @@ scenario name: _install agent name: _install cd suite && npx tsx src/runner.ts --agent={{name}} -# Open report in browser +# Open report in browser (honors GYM_OUTPUT_DIR if set) report: - open report.html + #!/usr/bin/env bash + set -euo pipefail + dir="${GYM_OUTPUT_DIR:-.}" + open "${dir/#\~/$HOME}/report.html" # Install all dependencies install: diff --git a/evals/open-model-gym/README.md b/evals/open-model-gym/README.md index c2e6c11df50e..ecb12122021e 100644 --- a/evals/open-model-gym/README.md +++ b/evals/open-model-gym/README.md @@ -267,6 +267,7 @@ Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log` | Command | Description | |---------|-------------| | `just run` | Full test run (3 reps each, worst kept) | +| `just run-clean` | Full run with artifacts isolated under `~/.goose/gym-runs/` | | `just test` | Quick run (1 rep each) | | `just scenario ` | Run specific scenario | | `just agent ` | Run specific agent | @@ -283,9 +284,36 @@ npx tsx src/runner.ts --run-count=5 # Don't auto-open browser npx tsx src/runner.ts --no-open + +# Redirect all run artifacts outside the repo (see Output below) +npx tsx src/runner.ts --output-dir=~/.goose/gym-runs/latest ``` ## Output - `report.html` ā€” Live-updating HTML matrix showing pass/fail status, duration, and validation details - `logs/` ā€” Full agent output logs for each run + +By default these (plus the cache, scratch workdir, and isolated agent config +roots `.goose-root/` / `.opencode-root/` / `.pi-root/`) are written inside the +gym directory. They're gitignored, but still pile up in your checkout ā€” awkward +if you want to run the bench regularly or from a worktree. + +To keep the repo clean, redirect **all** run artifacts to a single base +directory with the `GYM_OUTPUT_DIR` env var (or the `--output-dir=` flag). +`config.yaml` and `scenarios/` are still read from the repo. + +```bash +# Everything lands under a timestamped dir outside the repo (YYYYDDMMHHMMSS) +GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run + +# Convenience recipe that does the timestamping for you +just run-clean + +# View the report from a redirected run +GYM_OUTPUT_DIR=~/.goose/gym-runs/20261406101500 just report +``` + +> Note: the run cache lives under the output dir too, so a fresh timestamped +> dir means a fresh (cold) cache. Point `GYM_OUTPUT_DIR` at a stable directory +> if you want cache reuse across runs. diff --git a/evals/open-model-gym/suite/src/runner.ts b/evals/open-model-gym/suite/src/runner.ts index dc291fee8c3a..5ac786a3ac72 100644 --- a/evals/open-model-gym/suite/src/runner.ts +++ b/evals/open-model-gym/suite/src/runner.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import { mkdirSync, writeFileSync, rmSync, readdirSync, existsSync, copyFileSync } from "node:fs"; -import { join, basename, dirname } from "node:path"; +import { join, basename, dirname, resolve } from "node:path"; import { homedir } from "node:os"; -import { execSync } from "node:child_process"; +import { execSync, execFileSync } from "node:child_process"; import { parse, stringify } from "yaml"; import { readFileSync } from "node:fs"; import { createHash } from "node:crypto"; @@ -87,11 +87,47 @@ interface CacheIndex { entries: Record; } +// ============================================================================= +// Output directory resolution +// ============================================================================= +// All run artifacts (cache, isolated agent config roots, scratch workdir, logs, +// and the HTML report) live under a single base directory. By default this is +// the in-repo gym directory, so existing behavior is unchanged. Set the +// GYM_OUTPUT_DIR env var or pass --output-dir= to redirect everything +// outside the repo and keep your checkout clean, e.g.: +// +// GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run +// +// config.yaml and scenarios/ are inputs and always read from the repo. + +const SUITE_DIR = join(import.meta.dirname, ".."); // .../open-model-gym/suite +const GYM_DIR = join(import.meta.dirname, "../.."); // .../open-model-gym + +function expandHome(p: string): string { + return p === "~" || p.startsWith("~/") ? join(homedir(), p.slice(1)) : p; +} + +// Resolved output base, or null to fall back to the legacy in-repo locations. +const OUTPUT_DIR: string | null = (() => { + const flag = process.argv + .find((a) => a.startsWith("--output-dir=")) + ?.split("=")[1]; + const base = flag ?? process.env.GYM_OUTPUT_DIR; + // Resolve to an absolute path: runners exec with cwd set to the workdir, so a + // relative base would make prompt/log paths resolve against the wrong dir. + return base ? resolve(expandHome(base)) : null; +})(); + +// Resolve an artifact path under OUTPUT_DIR when set, else its legacy anchor. +function artifactPath(name: string, legacyAnchor: string): string { + return join(OUTPUT_DIR ?? legacyAnchor, name); +} + // ============================================================================= // Cache Utilities // ============================================================================= -const CACHE_DIR = join(import.meta.dirname, "../.cache"); +const CACHE_DIR = artifactPath(".cache", SUITE_DIR); const CACHE_INDEX_PATH = join(CACHE_DIR, "index.json"); const CACHE_LOGS_DIR = join(CACHE_DIR, "logs"); const CACHE_VERSION = 1; @@ -296,7 +332,7 @@ const PLATFORM_EXTENSIONS = new Set([ ]); // Isolated goose config directory -const GOOSE_ROOT = join(import.meta.dirname, "../.goose-root"); +const GOOSE_ROOT = artifactPath(".goose-root", SUITE_DIR); const GOOSE_CONFIG_DIR = join(GOOSE_ROOT, "config"); function generateGooseConfig(model: ModelConfig, runner: RunnerConfig): object { @@ -397,7 +433,7 @@ async function runGooseAgent( // ============================================================================= // Isolated opencode config directory -const OPENCODE_ROOT = join(import.meta.dirname, "../.opencode-root"); +const OPENCODE_ROOT = artifactPath(".opencode-root", SUITE_DIR); function generateOpenCodeConfig(model: ModelConfig, runner: RunnerConfig, workdir: string): object { const mcp: Record = {}; @@ -496,7 +532,7 @@ async function runOpenCodeAgent( // MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter` // Isolated Pi config directory (like Goose/OpenCode) -const PI_CONFIG_DIR = join(import.meta.dirname, "../.pi-root"); +const PI_CONFIG_DIR = artifactPath(".pi-root", SUITE_DIR); // User's real Pi config (for copying auth.json) const PI_USER_CONFIG = join(homedir(), ".pi", "agent"); @@ -979,11 +1015,15 @@ function generateHtmlReport( ): void { const { isRunning = false, allPairs = [] } = options; - // Read and embed gym.png as base64 - const rootDir = join(outputPath, ".."); + // Read and embed gym.png as base64. Prefer one sitting next to the report + // (legacy in-repo layout); otherwise fall back to the copy in the source tree + // so the image still embeds when output is redirected via GYM_OUTPUT_DIR. let gymBase64 = ""; try { - const gymPath = join(rootDir, "gym.png"); + const adjacent = join(outputPath, "..", "gym.png"); + const gymPath = existsSync(adjacent) + ? adjacent + : join(import.meta.dirname, "gym.png"); gymBase64 = readFileSync(gymPath).toString("base64"); } catch (e) { // gym.png not found, will use external reference @@ -1323,6 +1363,7 @@ function generateHtmlReport( `; + mkdirSync(dirname(outputPath), { recursive: true }); writeFileSync(outputPath, html); console.log(`\nšŸ“Š Report saved to: ${outputPath}`); } @@ -1361,12 +1402,11 @@ async function main() { return; } - const rootDir = join(import.meta.dirname, "../.."); - const configPath = join(rootDir, "config.yaml"); + const configPath = join(GYM_DIR, "config.yaml"); const scenariosDir = join(import.meta.dirname, "../scenarios"); - const workdir = join(import.meta.dirname, "../.workdir"); - const logsDir = join(rootDir, "logs"); - const reportPath = join(rootDir, "report.html"); + const workdir = artifactPath(".workdir", SUITE_DIR); + const logsDir = artifactPath("logs", GYM_DIR); + const reportPath = artifactPath("report.html", GYM_DIR); const config = loadConfig(configPath); let scenarios = loadAllScenarios(scenariosDir); @@ -1423,6 +1463,7 @@ async function main() { } const mcpHarnessHash = getMcpHarnessHash(); + console.log(`Output: ${OUTPUT_DIR ?? GYM_DIR}${OUTPUT_DIR ? "" : " (in-repo; set GYM_OUTPUT_DIR to redirect)"}`); console.log(`Models: ${config.models.map((m) => m.name).join(", ")}`); console.log(`Runners: ${config.runners.map((r) => r.name).join(", ")}`); console.log(`Running ${pairs.length} test pairs (${RUN_COUNT}x each, worst result kept)`); @@ -1455,7 +1496,7 @@ async function main() { if (!browserOpened) { generateHtmlReport(results, reportPath, { isRunning: true, allPairs: pairs }); if (!noOpen) { - execSync(`open "${reportPath}"`); + execFileSync("open", [reportPath]); } browserOpened = true; } @@ -1490,10 +1531,10 @@ async function main() { } generateHtmlReport(results, reportPath, { isRunning: false, allPairs: pairs }); - + // If everything was cached, open browser now with final report if (!browserOpened && !noOpen) { - execSync(`open "${reportPath}"`); + execFileSync("open", [reportPath]); } printResults(results);