fix: end the null harness cleanly on context overflow

verifiers/v1/harnesses/null/program.py

@@ -11,12 +11,43 @@
 from pathlib import Path
 
 import httpx
-from openai import AsyncOpenAI
+from openai import APIStatusError, AsyncOpenAI
 from tenacity import AsyncRetrying, stop_after_attempt, wait_exponential_jitter
 
 MCP_CALL_ATTEMPTS = 6
 MCP_TIMEOUT = 600.0
 
+CONTEXT_OVERFLOW_MARKERS = (
+    # OpenAI error code "context_length_exceeded"; OpenRouter relays the raw body.
+    "context_length_exceeded",
+    # OpenAI Responses/Completions: "Your input exceeds the context window of this model".
+    "exceeds the context window",
+    # OpenAI chat: "Input tokens exceed the configured limit of N tokens. Please reduce
+    # the length of the messages."; Groq words it the same way.
+    "reduce the length of the messages",
+    # vLLM: "This model's maximum context length is N tokens"; the renderers pre-flight:
+    # "Prompt length (N) exceeds maximum context length (M)"; Mistral uses the same words.
+    "maximum context length",
+    # Anthropic: "prompt is too long: N tokens > M maximum".
+    "prompt is too long",
+    # Anthropic byte-size overflow: HTTP 413 {"type": "request_too_large"}.
+    "request_too_large",
+    # HTTP proxies reject an oversized body with 413 "Request Entity Too Large".
+    "request entity too large",
+    # Google: "The input token count (N) exceeds the maximum number of tokens allowed (M)".
+    "exceeds the maximum number of tokens",
+    # xAI: "This model's maximum prompt length is N but the request contains M tokens".
+    "maximum prompt length is",
+)
+
+
+def is_context_overflow(error: APIStatusError) -> bool:
+    details = f"{error} {error.body or ''}".casefold()
+    # An overflow is deterministic: a 400, or a 413 for a byte-size cap.
+    return error.status_code in (400, 413) and any(
+        marker in details for marker in CONTEXT_OVERFLOW_MARKERS
+    )
+
 
 async def chat(
     client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict]
@@ -181,7 +212,14 @@ async def main() -> None:
     elif args.prompt:
         messages.append({"role": "user", "content": args.prompt})
     while True:
-        message = await chat(client, args.model, messages, tools)
+        try:
+            message = await chat(client, args.model, messages, tools)
+        except APIStatusError as error:
+            # Context exhaustion is a budget limit, not a crash: this harness has no
+            # compaction, so end the run cleanly with what the conversation holds.
+            if not is_context_overflow(error):
+                raise
+            return
         messages.append(message.model_dump(exclude_none=True))
         if not message.tool_calls:
             break

verifiers/v1/rollout.py

@@ -409,9 +409,9 @@ async def step(self, messages: Messages | None = None) -> bool:
                     0.0, self._agent_time_remaining - (loop.time() - segment_start)
                 )
             self.deadline_at = None
-        if self._session.error is not None:
-            self.fail(self._session.error)
-            return False
+        # A harness that completes cleanly after a failed model call handled it (e.g. it
+        # ends its run on context overflow); the failure stays recorded on the call. A
+        # harness that dies on it surfaces the stashed error through the except above.
         # A segment that committed nothing can't be waiting on the user; treating
         # it as continuable would consult the user against a conversation that
         # never moved, forever.

verifiers/v1/session.py

@@ -131,10 +131,10 @@ class RolloutSession:
     `register` (one server-owned client per distinct endpoint config), so every rollout it
     multiplexes shares one keepalive connection pool instead of opening its own."""
     error: "RolloutError | None" = None
-    """The latest unresolved model-call failure. The harness only sees it as an HTTP error
-    (and may swallow it, or exit non-zero), so the rollout re-raises this original error once the
-    harness returns — recording the real `ProviderError` instead of a secondary `HarnessError`.
-    Reset before each model turn, so a successful retry clears it."""
+    """The latest unresolved model-call failure. The harness only sees it as an HTTP error, so
+    when its program dies on it the rollout records this original error instead of a secondary
+    `HarnessError`. A harness that completes cleanly after the failure handled it. Reset before
+    each model turn, so a successful retry clears it."""
     idempotent_requests: dict[str, IdempotentRequest] = field(default_factory=dict)
     """Explicit keys or marked SDK retries mapped to their replay state."""
     released: bool = False