From 6e6e8ab01a830eaab39d2ff4d37e756793a344fe Mon Sep 17 00:00:00 2001
From: Kyle De Freitas <kdefreitas@squareup.com>
Date: Sun, 14 Jun 2026 19:46:09 -0400
Subject: [PATCH] feat(gym): make per-agent timeout configurable
 (GYM_AGENT_TIMEOUT)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The agent subprocess timeout was hardcoded to 5 minutes for all runners.
Larger local models (e.g. a 30B-class ollama model on heavier scenarios)
routinely exceed that and get SIGKILLed mid-task, which the harness then
records as a plain failure — indistinguishable from the model actually
failing the task. That silently penalizes slow-but-capable models.

Make the cap configurable via GYM_AGENT_TIMEOUT (seconds) or
--agent-timeout=<seconds>, applied to the goose/opencode/pi runners.
Default stays 300s, so existing behavior is unchanged.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Kyle De Freitas <kdefreitas@squareup.com>
---
 evals/open-model-gym/README.md           |  4 +++
 evals/open-model-gym/suite/src/runner.ts | 31 ++++++++++++++++++++----
 2 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/evals/open-model-gym/README.md b/evals/open-model-gym/README.md
index c2e6c11df50e..fb5e39197f19 100644
--- a/evals/open-model-gym/README.md
+++ b/evals/open-model-gym/README.md
@@ -283,6 +283,10 @@ npx tsx src/runner.ts --run-count=5
 
 # Don't auto-open browser
 npx tsx src/runner.ts --no-open
+
+# Raise the per-agent timeout (seconds) for slow local models on heavy
+# scenarios. Default 300s; also settable via GYM_AGENT_TIMEOUT.
+npx tsx src/runner.ts --agent-timeout=1200
 ```
 
 ## Output
diff --git a/evals/open-model-gym/suite/src/runner.ts b/evals/open-model-gym/suite/src/runner.ts
index dc291fee8c3a..124ff08709bc 100644
--- a/evals/open-model-gym/suite/src/runner.ts
+++ b/evals/open-model-gym/suite/src/runner.ts
@@ -66,6 +66,7 @@ interface CacheInputs {
   runnerHash: string;
   binaryHash: string;
   mcpHarnessHash: string;
+  timeoutMs: number;
 }
 
 interface CacheEntry {
@@ -87,6 +88,21 @@ interface CacheIndex {
   entries: Record<string, CacheEntry>;
 }
 
+// =============================================================================
+// Agent timeout
+// =============================================================================
+// Per-invocation timeout for an agent run, in milliseconds. Larger local models
+// can exceed the old fixed 5-minute cap on heavier scenarios and get killed
+// mid-task (recorded as a failure rather than a timeout). Override the cap with
+// GYM_AGENT_TIMEOUT (seconds) or --agent-timeout=<seconds>; default 300s.
+const AGENT_TIMEOUT_MS = (() => {
+  const flag = process.argv
+    .find((a) => a.startsWith("--agent-timeout="))
+    ?.split("=")[1];
+  const secs = parseInt(flag ?? process.env.GYM_AGENT_TIMEOUT ?? "", 10);
+  return (Number.isFinite(secs) && secs > 0 ? secs : 300) * 1000;
+})();
+
 // =============================================================================
 // Cache Utilities
 // =============================================================================
@@ -180,10 +196,15 @@ function computeCacheKey(pair: TestPair, binaryHashes: Map<string, string>, mcpH
     runnerHash,
     binaryHash,
     mcpHarnessHash,
+    timeoutMs: AGENT_TIMEOUT_MS,
   };
 
-  // Combine all into single key
-  const key = sha256(scenarioHash + modelKey + runnerHash + binaryHash + mcpHarnessHash);
+  // Combine all into single key. The timeout is included so a result cached
+  // under a short timeout (e.g. a 300s ETIMEDOUT) isn't reused when the run is
+  // retried with a larger GYM_AGENT_TIMEOUT.
+  const key = sha256(
+    scenarioHash + modelKey + runnerHash + binaryHash + mcpHarnessHash + AGENT_TIMEOUT_MS,
+  );
 
   return { key, inputs };
 }
@@ -385,7 +406,7 @@ async function runGooseAgent(
       GOOSE_PATH_ROOT: GOOSE_ROOT,
       MCP_HARNESS_LOG: join(workdir, "tool-calls.log"),
     },
-    timeout: 5 * 60 * 1000,
+    timeout: AGENT_TIMEOUT_MS,
     encoding: "utf-8",
   });
 
@@ -479,7 +500,7 @@ async function runOpenCodeAgent(
       XDG_CONFIG_HOME: OPENCODE_ROOT,
       XDG_DATA_HOME: OPENCODE_ROOT,
     },
-    timeout: 5 * 60 * 1000,
+    timeout: AGENT_TIMEOUT_MS,
     encoding: "utf-8",
     shell: "/bin/bash",
   });
@@ -644,7 +665,7 @@ async function runPiAgent(
       PI_CODING_AGENT_DIR: PI_CONFIG_DIR,  // Use isolated config dir
       MCP_HARNESS_LOG: join(workdir, "tool-calls.log"),
     },
-    timeout: 5 * 60 * 1000,
+    timeout: AGENT_TIMEOUT_MS,
     encoding: "utf-8",
     shell: "/bin/bash",
   });