feat(gym): make Open Model Gym output dir configurable

evals/open-model-gym/Justfile

@@ -7,6 +7,14 @@ default: run
 run: _install
     cd suite && npm run test
 
+# Full run with artifacts isolated under ~/.goose/gym-runs/<YYYYDDMMHHMMSS> (keeps the repo clean)
+run-clean: _install
+    #!/usr/bin/env bash
+    set -euo pipefail
+    export GYM_OUTPUT_DIR="$HOME/.goose/gym-runs/$(date +%Y%d%m%H%M%S)"
+    echo "Artifacts → $GYM_OUTPUT_DIR"
+    cd suite && npm run test
+
 # Quick test - file-editing + everyday-app-automation, single run each (no repetition)
 test: _install
     cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1
@@ -19,9 +27,12 @@ scenario name: _install
 agent name: _install
     cd suite && npx tsx src/runner.ts --agent={{name}}
 
-# Open report in browser
+# Open report in browser (honors GYM_OUTPUT_DIR if set)
 report:
-    open report.html
+    #!/usr/bin/env bash
+    set -euo pipefail
+    dir="${GYM_OUTPUT_DIR:-.}"
+    open "${dir/#\~/$HOME}/report.html"
 
 # Install all dependencies
 install:

evals/open-model-gym/README.md

@@ -267,6 +267,7 @@ Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log`
 | Command | Description |
 |---------|-------------|
 | `just run` | Full test run (3 reps each, worst kept) |
+| `just run-clean` | Full run with artifacts isolated under `~/.goose/gym-runs/<YYYYDDMMHHMMSS>` |
 | `just test` | Quick run (1 rep each) |
 | `just scenario <name>` | Run specific scenario |
 | `just agent <name>` | Run specific agent |
@@ -284,6 +285,9 @@ npx tsx src/runner.ts --run-count=5
 # Don't auto-open browser
 npx tsx src/runner.ts --no-open
 
+# Redirect all run artifacts outside the repo (see Output below)
+npx tsx src/runner.ts --output-dir=~/.goose/gym-runs/latest
+
 # Raise the per-agent timeout (seconds) for slow local models on heavy
 # scenarios. Default 300s; also settable via GYM_AGENT_TIMEOUT.
 npx tsx src/runner.ts --agent-timeout=1200
@@ -293,3 +297,27 @@ npx tsx src/runner.ts --agent-timeout=1200
 
 - `report.html` — Live-updating HTML matrix showing pass/fail status, duration, and validation details
 - `logs/` — Full agent output logs for each run
+
+By default these (plus the cache, scratch workdir, and isolated agent config
+roots `.goose-root/` / `.opencode-root/` / `.pi-root/`) are written inside the
+gym directory. They're gitignored, but still pile up in your checkout — awkward
+if you want to run the bench regularly or from a worktree.
+
+To keep the repo clean, redirect **all** run artifacts to a single base
+directory with the `GYM_OUTPUT_DIR` env var (or the `--output-dir=` flag).
+`config.yaml` and `scenarios/` are still read from the repo.
+
+```bash
+# Everything lands under a timestamped dir outside the repo (YYYYDDMMHHMMSS)
+GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run
+
+# Convenience recipe that does the timestamping for you
+just run-clean
+
+# View the report from a redirected run
+GYM_OUTPUT_DIR=~/.goose/gym-runs/20261406101500 just report
+```
+
+> Note: the run cache lives under the output dir too, so a fresh timestamped
+> dir means a fresh (cold) cache. Point `GYM_OUTPUT_DIR` at a stable directory
+> if you want cache reuse across runs.

evals/open-model-gym/suite/src/runner.ts

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { mkdirSync, writeFileSync, rmSync, readdirSync, existsSync, copyFileSync } from "node:fs";
-import { join, basename, dirname } from "node:path";
+import { join, basename, dirname, resolve } from "node:path";
 import { homedir } from "node:os";
-import { execSync } from "node:child_process";
+import { execSync, execFileSync } from "node:child_process";
 import { parse, stringify } from "yaml";
 import { readFileSync } from "node:fs";
 import { createHash } from "node:crypto";
@@ -89,6 +89,41 @@ interface CacheIndex {
 }
 
 // =============================================================================
+// Output directory resolution
+// =============================================================================
+// All run artifacts (cache, isolated agent config roots, scratch workdir, logs,
+// and the HTML report) live under a single base directory. By default this is
+// the in-repo gym directory, so existing behavior is unchanged. Set the
+// GYM_OUTPUT_DIR env var or pass --output-dir=<path> to redirect everything
+// outside the repo and keep your checkout clean, e.g.:
+//
+//   GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run
+//
+// config.yaml and scenarios/ are inputs and always read from the repo.
+
+const SUITE_DIR = join(import.meta.dirname, ".."); // .../open-model-gym/suite
+const GYM_DIR = join(import.meta.dirname, "../.."); // .../open-model-gym
+
+function expandHome(p: string): string {
+  return p === "~" || p.startsWith("~/") ? join(homedir(), p.slice(1)) : p;
+}
+
+// Resolved output base, or null to fall back to the legacy in-repo locations.
+const OUTPUT_DIR: string | null = (() => {
+  const flag = process.argv
+    .find((a) => a.startsWith("--output-dir="))
+    ?.split("=")[1];
+  const base = flag ?? process.env.GYM_OUTPUT_DIR;
+  // Resolve to an absolute path: runners exec with cwd set to the workdir, so a
+  // relative base would make prompt/log paths resolve against the wrong dir.
+  return base ? resolve(expandHome(base)) : null;
+})();
+
+// Resolve an artifact path under OUTPUT_DIR when set, else its legacy anchor.
+function artifactPath(name: string, legacyAnchor: string): string {
+  return join(OUTPUT_DIR ?? legacyAnchor, name);
+}
+
 // Agent timeout
 // =============================================================================
 // Per-invocation timeout for an agent run, in milliseconds. Larger local models
@@ -107,7 +142,7 @@ const AGENT_TIMEOUT_MS = (() => {
 // Cache Utilities
 // =============================================================================
 
-const CACHE_DIR = join(import.meta.dirname, "../.cache");
+const CACHE_DIR = artifactPath(".cache", SUITE_DIR);
 const CACHE_INDEX_PATH = join(CACHE_DIR, "index.json");
 const CACHE_LOGS_DIR = join(CACHE_DIR, "logs");
 const CACHE_VERSION = 1;
@@ -317,7 +352,7 @@ const PLATFORM_EXTENSIONS = new Set([
 ]);
 
 // Isolated goose config directory
-const GOOSE_ROOT = join(import.meta.dirname, "../.goose-root");
+const GOOSE_ROOT = artifactPath(".goose-root", SUITE_DIR);
 const GOOSE_CONFIG_DIR = join(GOOSE_ROOT, "config");
 
 function generateGooseConfig(model: ModelConfig, runner: RunnerConfig): object {
@@ -418,7 +453,7 @@ async function runGooseAgent(
 // =============================================================================
 
 // Isolated opencode config directory
-const OPENCODE_ROOT = join(import.meta.dirname, "../.opencode-root");
+const OPENCODE_ROOT = artifactPath(".opencode-root", SUITE_DIR);
 
 function generateOpenCodeConfig(model: ModelConfig, runner: RunnerConfig, workdir: string): object {
   const mcp: Record<string, object> = {};
@@ -517,7 +552,7 @@ async function runOpenCodeAgent(
 // MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
 
 // Isolated Pi config directory (like Goose/OpenCode)
-const PI_CONFIG_DIR = join(import.meta.dirname, "../.pi-root");
+const PI_CONFIG_DIR = artifactPath(".pi-root", SUITE_DIR);
 
 // User's real Pi config (for copying auth.json)
 const PI_USER_CONFIG = join(homedir(), ".pi", "agent");
@@ -1000,11 +1035,15 @@ function generateHtmlReport(
 ): void {
   const { isRunning = false, allPairs = [] } = options;
 
-  // Read and embed gym.png as base64
-  const rootDir = join(outputPath, "..");
+  // Read and embed gym.png as base64. Prefer one sitting next to the report
+  // (legacy in-repo layout); otherwise fall back to the copy in the source tree
+  // so the image still embeds when output is redirected via GYM_OUTPUT_DIR.
   let gymBase64 = "";
   try {
-    const gymPath = join(rootDir, "gym.png");
+    const adjacent = join(outputPath, "..", "gym.png");
+    const gymPath = existsSync(adjacent)
+      ? adjacent
+      : join(import.meta.dirname, "gym.png");
     gymBase64 = readFileSync(gymPath).toString("base64");
   } catch (e) {
     // gym.png not found, will use external reference
@@ -1344,6 +1383,7 @@ function generateHtmlReport(
 </body>
 </html>`;
 
+  mkdirSync(dirname(outputPath), { recursive: true });
   writeFileSync(outputPath, html);
   console.log(`\n📊 Report saved to: ${outputPath}`);
 }
@@ -1382,12 +1422,11 @@ async function main() {
     return;
   }
 
-  const rootDir = join(import.meta.dirname, "../..");
-  const configPath = join(rootDir, "config.yaml");
+  const configPath = join(GYM_DIR, "config.yaml");
   const scenariosDir = join(import.meta.dirname, "../scenarios");
-  const workdir = join(import.meta.dirname, "../.workdir");
-  const logsDir = join(rootDir, "logs");
-  const reportPath = join(rootDir, "report.html");
+  const workdir = artifactPath(".workdir", SUITE_DIR);
+  const logsDir = artifactPath("logs", GYM_DIR);
+  const reportPath = artifactPath("report.html", GYM_DIR);
 
   const config = loadConfig(configPath);
   let scenarios = loadAllScenarios(scenariosDir);
@@ -1444,6 +1483,7 @@ async function main() {
   }
   const mcpHarnessHash = getMcpHarnessHash();
 
+  console.log(`Output: ${OUTPUT_DIR ?? GYM_DIR}${OUTPUT_DIR ? "" : " (in-repo; set GYM_OUTPUT_DIR to redirect)"}`);
   console.log(`Models: ${config.models.map((m) => m.name).join(", ")}`);
   console.log(`Runners: ${config.runners.map((r) => r.name).join(", ")}`);
   console.log(`Running ${pairs.length} test pairs (${RUN_COUNT}x each, worst result kept)`);
@@ -1476,7 +1516,7 @@ async function main() {
     if (!browserOpened) {
       generateHtmlReport(results, reportPath, { isRunning: true, allPairs: pairs });
       if (!noOpen) {
-        execSync(`open "${reportPath}"`);
+        execFileSync("open", [reportPath]);
       }
       browserOpened = true;
     }
@@ -1511,10 +1551,10 @@ async function main() {
   }
 
   generateHtmlReport(results, reportPath, { isRunning: false, allPairs: pairs });
-  
+
   // If everything was cached, open browser now with final report
   if (!browserOpened && !noOpen) {
-    execSync(`open "${reportPath}"`);
+    execFileSync("open", [reportPath]);
   }
 
   printResults(results);