From 1ac73f4229ef461bbb8aa4c18b8591d82385df7e Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 19:29:44 +0000 Subject: [PATCH 01/34] feat(v1): record raw per-call request/response on the trace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every provider exchange behind a sampled turn lands on Trace.calls as an untyped ModelCall: the request body as actually sent upstream (model + sampling overrides applied), the native response object, provider response headers, wall-clock span, time-to-first-token for streamed turns, and — for a failed exchange — the error, coupled to the call that raised it. Each successful call links into the message graph via the id of the assistant node it committed (PendingTurn.commit now returns it), so per-call data joins the graph exactly even under branching (compaction, retokenized prefixes). Replayed/coalesced SDK retries record nothing — they are not provider exchanges. Closes RES-1085. Co-Authored-By: Claude Fable 5 --- tests/v1/test_e2e.py | 8 +++ verifiers/v1/clients/client.py | 2 + verifiers/v1/clients/eval.py | 2 + verifiers/v1/dialects/anthropic.py | 4 +- verifiers/v1/dialects/chat.py | 31 ++++---- verifiers/v1/dialects/responses.py | 4 +- verifiers/v1/graph.py | 10 ++- verifiers/v1/interception/server.py | 106 ++++++++++++++++++++++++++-- verifiers/v1/trace.py | 34 +++++++++ verifiers/v1/types.py | 6 +- 10 files changed, 181 insertions(+), 26 deletions(-) diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index f915082a57..eda519951c 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -18,6 +18,14 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path): assert trace.errors == [] assert trace.num_turns == 1 assert trace.reward == 1.0 + # Every sampled turn has one raw per-call record, linked to its assistant node. + sampled = [i for i, n in enumerate(trace.nodes) if n.sampled] + assert [c.node for c in trace.calls if c.error is None] == sampled + for call in trace.calls: + assert call.request is not None + assert call.time.duration > 0 + if call.error is None: + assert call.response is not None @pytest.mark.e2e diff --git a/verifiers/v1/clients/client.py b/verifiers/v1/clients/client.py index 04b6adbfb3..70bf59a6ad 100644 --- a/verifiers/v1/clients/client.py +++ b/verifiers/v1/clients/client.py @@ -26,6 +26,8 @@ class RelayReply: content_type: str chunks: AsyncIterator[bytes] close: Callable[[], Awaitable[None]] + headers: dict[str, str] | None = None + """Provider response headers, for the trace's per-call records.""" class Client(ABC): diff --git a/verifiers/v1/clients/eval.py b/verifiers/v1/clients/eval.py index 6a7df3369c..bd3991fa88 100644 --- a/verifiers/v1/clients/eval.py +++ b/verifiers/v1/clients/eval.py @@ -110,6 +110,7 @@ async def get_response( ) from e # The interception server returns this full native provider object to the program. response.raw = raw + response.raw_headers = dict(resp.headers) return response def _headers( @@ -217,6 +218,7 @@ async def chunks(): content_type=resp.headers.get("content-type", "text/event-stream"), chunks=chunks(), close=resp.aclose, + headers=dict(resp.headers), ) async def relay_aux( diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 254bc2a9c1..597e076a7c 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -239,7 +239,9 @@ def finish(self) -> Response: for index, parts in self.partial_json.items(): self.blocks[index]["input"] = json.loads("".join(parts) or "{}") self.message["content"] = [self.blocks[index] for index in sorted(self.blocks)] - return response_from_wire(self.validate_response(self.message)) + response = response_from_wire(self.validate_response(self.message)) + response.raw = self.message + return response class AnthropicDialect(Dialect[dict, AnthropicMessage]): diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index a2aff6c992..531c45df8b 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -271,24 +271,23 @@ def finish(self) -> Response: if self.reasoning_details: self.message["reasoning_details"] = self.reasoning_details head = self.head or {} - return response_from_wire( - ModdedChatCompletion.model_validate( + completion = { + "id": head.get("id", "vf-intercept"), + "object": "chat.completion", + "created": head.get("created", int(time.time())), + "model": head.get("model", ""), + "choices": [ { - "id": head.get("id", "vf-intercept"), - "object": "chat.completion", - "created": head.get("created", int(time.time())), - "model": head.get("model", ""), - "choices": [ - { - "index": 0, - "message": self.message, - "finish_reason": self.finish_reason or "stop", - } - ], - "usage": self.usage, + "index": 0, + "message": self.message, + "finish_reason": self.finish_reason or "stop", } - ) - ) + ], + "usage": self.usage, + } + response = response_from_wire(ModdedChatCompletion.model_validate(completion)) + response.raw = completion + return response class ChatDialect(Dialect[dict, ChatCompletion]): diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index c81c06a197..74bbaa27ca 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -270,9 +270,11 @@ def finish(self) -> Response: events = self.terminal_events or self.events for event in iter_sse_reverse(b"".join(events)): if event.get("type") in FINAL_EVENTS: - return response_from_wire( + response = response_from_wire( OpenAIResponse.model_validate(event["response"]) ) + response.raw = event["response"] + return response raise ValueError("Responses stream ended without a terminal event") diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index 6d0156bbf7..8f2222acad 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -343,10 +343,12 @@ def prompt_message_spans( for span in tail_spans ] - def commit(self, response: Response, tools: list[Tool] | None = None) -> None: - _commit_turn(self, response) + def commit(self, response: Response, tools: list[Tool] | None = None) -> int: + """Add this turn to the graph; returns the committed assistant node's id.""" + assistant_id = _commit_turn(self, response) if tools: self.trace.tools = tools + return assistant_id def prepare_turn(trace: Trace, prompt: list[Message]) -> PendingTurn: @@ -486,7 +488,7 @@ def _attribute_kept_tokens( node.kept_tokens = KeptTokens(ids=ids.copy(), counts=counts.copy()) -def _commit_turn(turn: PendingTurn, response: Response) -> None: +def _commit_turn(turn: PendingTurn, response: Response) -> int: trace = turn.trace prompt = turn.prompt tokens = response.tokens @@ -596,6 +598,8 @@ def _commit_turn(turn: PendingTurn, response: Response) -> None: # completion-aligned, so only the sampled node carries them). _attribute_kept_tokens(trace, assistant_id, tokens.kept_tokens if tokens else None) + return assistant_id + # --- walking the graph (views) --------------------------------------------------------- diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 4540a6f2f5..b1a8507523 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -25,6 +25,8 @@ import json import logging import secrets +import time +import traceback from collections.abc import AsyncIterator from contextlib import asynccontextmanager from typing import Literal @@ -38,6 +40,7 @@ from verifiers.v1 import graph from verifiers.v1.errors import ( OverlongPromptError, + ProviderError, RolloutError, TaskError, UserError, @@ -50,6 +53,7 @@ make_tunnel, ) from verifiers.v1.session import RolloutSession +from verifiers.v1.trace import Error, ModelCall, TimeSpan from verifiers.v1.types import Messages, Response, Tool logger = logging.getLogger(__name__) @@ -229,6 +233,45 @@ def _fail( status=getattr(error, "status_code", 502), ) + def _record_call( + self, + session: RolloutSession, + dialect: Dialect, + request: dict, + started: float, + *, + node: int | None = None, + response: dict | None = None, + headers: dict[str, str] | None = None, + first_token: float | None = None, + error: Exception | None = None, + ) -> None: + """Append one provider exchange to the trace's per-call records (`Trace.calls`): + the raw request as sent upstream, the raw native response, timing, and — when the + call committed no turn — the error, coupled to the exchange that raised it. Called + once per real exchange; replayed/coalesced SDK retries never reach it.""" + session.trace.calls.append( + ModelCall( + node=node, + endpoint=dialect.upstream_path, + request=request, + response=response, + response_headers=headers, + time=TimeSpan(start=started, end=time.time()), + time_to_first_token=first_token, + error=None + if error is None + else Error( + type=type(error).__name__, + message=str(error), + # Provider errors already carry the actionable upstream diagnostic. + traceback=None + if isinstance(error, ProviderError) + else traceback.format_exc(), + ), + ) + ) + async def handle_request( self, request: web.Request, dialect: Dialect ) -> web.StreamResponse: @@ -352,6 +395,12 @@ def serve(response: Response) -> web.Response: return serve(response) turn = graph.prepare_turn(session.trace, prompt) session.error = None + # What actually goes upstream: the native body with the rollout's model + + # sampling imposed — recorded raw on the trace, per call. + upstream_request = dialect.apply_overrides( + body, session.ctx.model, session.ctx.sampling + ) + started = time.time() try: response = await session.ctx.client.get_response( dialect, @@ -362,10 +411,13 @@ def serve(response: Response) -> web.Response: session_id=session.trace.id, turn=turn, ) - except OverlongPromptError: + except OverlongPromptError as e: # An overlong prompt is a budget limit, not a crash: end the rollout cleanly # as a truncation — return the last turn if there is one, else refuse to halt # the harness (same shape as `refused` above). + self._record_call( + session, dialect, upstream_request, started, error=e + ) session.trace.stop("context_length") logger.debug("prompt too long: id=%s", session.trace.id) if response is None: @@ -377,6 +429,9 @@ def serve(response: Response) -> web.Response: except RolloutError as e: # Stash the real cause; the rollout re-raises it after the harness returns. # Relay the provider's status so the harness SDK retries 5xx/429 and not 4xx. + self._record_call( + session, dialect, upstream_request, started, error=e + ) session.error = e logger.warning( "model call failed: id=%s %s: %s", @@ -389,6 +444,9 @@ def serve(response: Response) -> web.Response: status=getattr(e, "status_code", 502), ) except Exception as e: # surface to the program as an API error + self._record_call( + session, dialect, upstream_request, started, error=e + ) logger.warning( "model call failed: id=%s %s: %s", session.trace.id, @@ -401,8 +459,17 @@ def serve(response: Response) -> web.Response: session.trace.id, len(response.message.tool_calls or []), ) - turn.commit(response, tools) # one node per new message; + node = turn.commit(response, tools) # one node per new message; # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph) + self._record_call( + session, + dialect, + upstream_request, + started, + node=node, + response=response.raw, + headers=response.raw_headers, + ) # Hand back to the program when the model wants a tool (the program runs it) or # when there's no user simulator to keep the conversation going. if response.message.tool_calls or session.user is None: @@ -462,6 +529,10 @@ async def _stream( dialect.error_body(f"rollout stopped: {refused}"), status=400 ) session.error = None + upstream_request = dialect.apply_overrides( + body, session.ctx.model, session.ctx.sampling + ) + started = time.time() try: turn = graph.prepare_turn(session.trace, prompt) reply = await session.ctx.client.relay( @@ -472,13 +543,15 @@ async def _stream( headers=request.headers, session_id=session.trace.id, ) - except OverlongPromptError: + except OverlongPromptError as e: + self._record_call(session, dialect, upstream_request, started, error=e) session.trace.stop("context_length") logger.debug("prompt too long: id=%s", session.trace.id) return web.json_response( dialect.error_body("rollout stopped: context_length"), status=400 ) except RolloutError as e: + self._record_call(session, dialect, upstream_request, started, error=e) session.error = e logger.warning( "model call failed: id=%s %s: %s", @@ -490,6 +563,7 @@ async def _stream( dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) except Exception as e: # surface to the program as an API error + self._record_call(session, dialect, upstream_request, started, error=e) logger.warning("model call failed: id=%s %s", session.trace.id, e) return web.json_response(dialect.error_body(str(e)), status=502) resp = web.StreamResponse( @@ -507,6 +581,7 @@ async def _stream( ready = asyncio.Event() producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready)) parser_error: Exception | None = None + first_token: float | None = None # SSE events from the turn-ending one onward (the terminal event and any trailing # `[DONE]`), withheld until the turn is committed: a client that ends its turn on the # terminal event (e.g. codex on `response.completed`) would otherwise reach scoring @@ -527,6 +602,8 @@ async def _stream( if chunk is None: await producer break + if first_token is None: + first_token = time.time() - started if deferred or dialect.is_terminal_event(chunk): if parser_error is None: try: @@ -557,8 +634,29 @@ async def _stream( try: if parser_error is not None: raise parser_error - turn.commit(parser.finish(), tools) + response = parser.finish() + node = turn.commit(response, tools) + self._record_call( + session, + dialect, + upstream_request, + started, + node=node, + response=response.raw, + headers=reply.headers, + first_token=first_token, + ) logger.debug("intercept stream turn: id=%s", session.trace.id) + except Exception as e: + self._record_call( + session, + dialect, + upstream_request, + started, + first_token=first_token, + error=e, + ) + raise finally: # Release the withheld events only now — after the commit — then close. with contextlib.suppress(ConnectionResetError): diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index e7fd537995..cb9ca23d3e 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -62,6 +62,37 @@ class Error(StrictBaseModel): traceback: str | None = None +class ModelCall(StrictBaseModel): + """One provider exchange behind a sampled turn, kept raw: the request as sent upstream + and the native response, untyped. Recorded by the interception server for every real + exchange — an SDK-level retry that replays or coalesces onto an earlier attempt adds + nothing, a failed attempt is recorded with its `error`.""" + + node: int | None = None + """Index into `Trace.nodes` of the assistant node this call committed — the link into + the message graph (the call's conversation is that node's root-to-self path). None for + a call that committed no turn (see `error`).""" + endpoint: str | None = None + """The provider endpoint path the request went to (e.g. `/chat/completions`) — says + which wire format `request` and `response` are in.""" + request: dict[str, Any] | None = None + """The raw request body as sent upstream: the harness's native JSON with the rollout's + model + sampling overrides applied — so it carries the effective sampling parameters + and the requested model.""" + response: dict[str, Any] | None = None + """The raw native response object (for a generating client, the completion it + synthesized): provider response id, returned model, native usage. None for a failed call.""" + response_headers: dict[str, str] | None = None + """Provider response headers (request ids, rate limits), when the transport exposes them.""" + time: TimeSpan = Field(default_factory=TimeSpan) + """Wall-clock span from sending the request to the fully received response.""" + time_to_first_token: float | None = None + """Seconds from sending the request to the first streamed event; None when not streaming.""" + error: Error | None = None + """The failure that ended this call, coupled to the exchange that caused it; None on + success. A failed call still records the request it sent.""" + + class Branch(StrictBaseModel): """A root-to-leaf graph path; each branch becomes one training sample.""" @@ -305,6 +336,9 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]): committed turn wins) — never from a refused/failed request the model never saw. The full advertised list (not just tools called), so tool-use SFT can re-render the exact prompt; a trace-level snapshot: mid-rollout changes collapse to the last set the model saw.""" + calls: list[ModelCall] = Field(default_factory=list) + """Every provider exchange behind the sampled turns, in order: raw wire request/response + plus per-call timing and errors, linked into `nodes` via `ModelCall.node`.""" rewards: dict[str, float] = Field(default_factory=dict) """Weighted contributions from task rewards, group rewards, and judges.""" diff --git a/verifiers/v1/types.py b/verifiers/v1/types.py index 7ee25f8502..a3b62712cc 100644 --- a/verifiers/v1/types.py +++ b/verifiers/v1/types.py @@ -229,7 +229,11 @@ class Response(StrictBaseModel): usage: Usage | None = None tokens: TurnTokens | None = None raw: dict | None = Field(default=None, exclude=True, repr=False) - """Full native response object returned to the program; excluded from traces.""" + """Full native response object returned to the program; recorded raw on the trace's + per-call records (`Trace.calls`), never dumped as part of the typed response.""" + raw_headers: dict[str, str] | None = Field(default=None, exclude=True, repr=False) + """Provider response headers (request ids, rate limits), when the transport exposes + them; carried to the trace's per-call records like `raw`.""" class SamplingConfig(BaseModel): From f656257b202b5a645f8cac463241946b3ebc3955 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 20:06:46 +0000 Subject: [PATCH 02/34] refactor: drop TTFT, type the call's dialect, unhide record_call time_to_first_token was only measurable on the streamed relay path; drop it until there's a consumer. The call's wire format is now a typed DialectName literal (chat/responses/anthropic, exported by v1.dialects) instead of the provider endpoint path. Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/__init__.py | 3 ++- verifiers/v1/dialects/anthropic.py | 1 + verifiers/v1/dialects/base.py | 8 ++++++- verifiers/v1/dialects/chat.py | 1 + verifiers/v1/dialects/responses.py | 1 + verifiers/v1/interception/server.py | 35 +++++++++-------------------- verifiers/v1/trace.py | 8 +++---- 7 files changed, 26 insertions(+), 31 deletions(-) diff --git a/verifiers/v1/dialects/__init__.py b/verifiers/v1/dialects/__init__.py index 72fd582f34..70664d27fc 100644 --- a/verifiers/v1/dialects/__init__.py +++ b/verifiers/v1/dialects/__init__.py @@ -1,7 +1,7 @@ """Registered wire dialects for interception.""" from verifiers.v1.dialects.anthropic import AnthropicDialect -from verifiers.v1.dialects.base import Dialect, StreamParser, iter_sse +from verifiers.v1.dialects.base import Dialect, DialectName, StreamParser, iter_sse from verifiers.v1.dialects.chat import ( FINISH_REASONS, ChatDialect, @@ -17,6 +17,7 @@ __all__ = [ "Dialect", + "DialectName", "DIALECTS", "FINISH_REASONS", "AnthropicDialect", diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 597e076a7c..1e1473e04b 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -245,6 +245,7 @@ def finish(self) -> Response: class AnthropicDialect(Dialect[dict, AnthropicMessage]): + name = "anthropic" routes = ("/v1/messages",) aux_routes = ("/v1/messages/count_tokens",) upstream_path = "/v1/messages" diff --git a/verifiers/v1/dialects/base.py b/verifiers/v1/dialects/base.py index 7846978317..dfc6195045 100644 --- a/verifiers/v1/dialects/base.py +++ b/verifiers/v1/dialects/base.py @@ -16,7 +16,7 @@ import logging from abc import ABC, abstractmethod from collections.abc import Callable, Iterator, Mapping -from typing import ClassVar, Generic, TypeVar +from typing import ClassVar, Generic, Literal, TypeVar from pydantic import BaseModel from pydantic_core import from_json @@ -26,6 +26,9 @@ ReqT = TypeVar("ReqT") RespT = TypeVar("RespT", bound=BaseModel) +DialectName = Literal["chat", "responses", "anthropic"] +"""The registered wire formats — how a trace's raw per-call `request`/`response` are shaped.""" + logger = logging.getLogger(__name__) @@ -121,6 +124,9 @@ class Dialect(ABC, Generic[ReqT, RespT]): `dialects.DIALECTS` and a harness speaking that format works end-to-end (the eval client and interception server are generic over this interface).""" + name: ClassVar[DialectName] + """This dialect's wire-format name, recorded on the trace's per-call records.""" + routes: ClassVar[tuple[str, ...]] """The endpoint path(s) a program's SDK posts model turns to. The interception server serves one handler per route, so the wire format is resolved from the route the SDK chose (it diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index 531c45df8b..8e94d89de5 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -291,6 +291,7 @@ def finish(self) -> Response: class ChatDialect(Dialect[dict, ChatCompletion]): + name = "chat" routes = ("/v1/chat/completions",) upstream_path = "/chat/completions" response_type = ModdedChatCompletion diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index 74bbaa27ca..fab9e53cb5 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -279,6 +279,7 @@ def finish(self) -> Response: class ResponsesDialect(Dialect[dict, OpenAIResponse]): + name = "responses" routes = ("/v1/responses",) upstream_path = "/responses" response_type = OpenAIResponse diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index b1a8507523..d784c4b4cc 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -233,7 +233,7 @@ def _fail( status=getattr(error, "status_code", 502), ) - def _record_call( + def record_call( self, session: RolloutSession, dialect: Dialect, @@ -243,7 +243,6 @@ def _record_call( node: int | None = None, response: dict | None = None, headers: dict[str, str] | None = None, - first_token: float | None = None, error: Exception | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): @@ -253,12 +252,11 @@ def _record_call( session.trace.calls.append( ModelCall( node=node, - endpoint=dialect.upstream_path, + dialect=dialect.name, request=request, response=response, response_headers=headers, time=TimeSpan(start=started, end=time.time()), - time_to_first_token=first_token, error=None if error is None else Error( @@ -415,7 +413,7 @@ def serve(response: Response) -> web.Response: # An overlong prompt is a budget limit, not a crash: end the rollout cleanly # as a truncation — return the last turn if there is one, else refuse to halt # the harness (same shape as `refused` above). - self._record_call( + self.record_call( session, dialect, upstream_request, started, error=e ) session.trace.stop("context_length") @@ -429,7 +427,7 @@ def serve(response: Response) -> web.Response: except RolloutError as e: # Stash the real cause; the rollout re-raises it after the harness returns. # Relay the provider's status so the harness SDK retries 5xx/429 and not 4xx. - self._record_call( + self.record_call( session, dialect, upstream_request, started, error=e ) session.error = e @@ -444,7 +442,7 @@ def serve(response: Response) -> web.Response: status=getattr(e, "status_code", 502), ) except Exception as e: # surface to the program as an API error - self._record_call( + self.record_call( session, dialect, upstream_request, started, error=e ) logger.warning( @@ -461,7 +459,7 @@ def serve(response: Response) -> web.Response: ) node = turn.commit(response, tools) # one node per new message; # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph) - self._record_call( + self.record_call( session, dialect, upstream_request, @@ -544,14 +542,14 @@ async def _stream( session_id=session.trace.id, ) except OverlongPromptError as e: - self._record_call(session, dialect, upstream_request, started, error=e) + self.record_call(session, dialect, upstream_request, started, error=e) session.trace.stop("context_length") logger.debug("prompt too long: id=%s", session.trace.id) return web.json_response( dialect.error_body("rollout stopped: context_length"), status=400 ) except RolloutError as e: - self._record_call(session, dialect, upstream_request, started, error=e) + self.record_call(session, dialect, upstream_request, started, error=e) session.error = e logger.warning( "model call failed: id=%s %s: %s", @@ -563,7 +561,7 @@ async def _stream( dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) except Exception as e: # surface to the program as an API error - self._record_call(session, dialect, upstream_request, started, error=e) + self.record_call(session, dialect, upstream_request, started, error=e) logger.warning("model call failed: id=%s %s", session.trace.id, e) return web.json_response(dialect.error_body(str(e)), status=502) resp = web.StreamResponse( @@ -581,7 +579,6 @@ async def _stream( ready = asyncio.Event() producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready)) parser_error: Exception | None = None - first_token: float | None = None # SSE events from the turn-ending one onward (the terminal event and any trailing # `[DONE]`), withheld until the turn is committed: a client that ends its turn on the # terminal event (e.g. codex on `response.completed`) would otherwise reach scoring @@ -602,8 +599,6 @@ async def _stream( if chunk is None: await producer break - if first_token is None: - first_token = time.time() - started if deferred or dialect.is_terminal_event(chunk): if parser_error is None: try: @@ -636,7 +631,7 @@ async def _stream( raise parser_error response = parser.finish() node = turn.commit(response, tools) - self._record_call( + self.record_call( session, dialect, upstream_request, @@ -644,18 +639,10 @@ async def _stream( node=node, response=response.raw, headers=reply.headers, - first_token=first_token, ) logger.debug("intercept stream turn: id=%s", session.trace.id) except Exception as e: - self._record_call( - session, - dialect, - upstream_request, - started, - first_token=first_token, - error=e, - ) + self.record_call(session, dialect, upstream_request, started, error=e) raise finally: # Release the withheld events only now — after the commit — then close. diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index cb9ca23d3e..2ea09013c1 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -15,6 +15,7 @@ from verifiers.v1.judge import JudgeResponse from verifiers.v1 import graph +from verifiers.v1.dialects import DialectName from verifiers.v1.errors import ProviderError from verifiers.v1.graph import MessageNode from verifiers.v1.harness import HarnessConfig @@ -72,9 +73,8 @@ class ModelCall(StrictBaseModel): """Index into `Trace.nodes` of the assistant node this call committed — the link into the message graph (the call's conversation is that node's root-to-self path). None for a call that committed no turn (see `error`).""" - endpoint: str | None = None - """The provider endpoint path the request went to (e.g. `/chat/completions`) — says - which wire format `request` and `response` are in.""" + dialect: DialectName | None = None + """The wire format `request` and `response` are shaped as.""" request: dict[str, Any] | None = None """The raw request body as sent upstream: the harness's native JSON with the rollout's model + sampling overrides applied — so it carries the effective sampling parameters @@ -86,8 +86,6 @@ class ModelCall(StrictBaseModel): """Provider response headers (request ids, rate limits), when the transport exposes them.""" time: TimeSpan = Field(default_factory=TimeSpan) """Wall-clock span from sending the request to the fully received response.""" - time_to_first_token: float | None = None - """Seconds from sending the request to the first streamed event; None when not streaming.""" error: Error | None = None """The failure that ended this call, coupled to the exchange that caused it; None on success. A failed call still records the request it sent.""" From e1ff38d7532cf288bc45260907c8267fd02e2cee Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 20:14:19 +0000 Subject: [PATCH 03/34] fix(v1): record mid-relay and pre-send call failures Two per-call recording gaps from review: a provider stream dying mid-relay propagated out of the pump loop without landing on Trace.calls, and apply_overrides ran outside the guarded try, so a malformed native field escaped as an unshaped 500 with no record. Both now record the failed exchange (request=None when the overrides themselves failed). Co-Authored-By: Claude Fable 5 --- verifiers/v1/interception/server.py | 25 ++++++++++++++++--------- verifiers/v1/trace.py | 4 +++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index d784c4b4cc..25bb70b506 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -237,7 +237,7 @@ def record_call( self, session: RolloutSession, dialect: Dialect, - request: dict, + request: dict | None, started: float, *, node: int | None = None, @@ -393,13 +393,14 @@ def serve(response: Response) -> web.Response: return serve(response) turn = graph.prepare_turn(session.trace, prompt) session.error = None - # What actually goes upstream: the native body with the rollout's model + - # sampling imposed — recorded raw on the trace, per call. - upstream_request = dialect.apply_overrides( - body, session.ctx.model, session.ctx.sampling - ) + upstream_request: dict | None = None started = time.time() try: + # What actually goes upstream: the native body with the rollout's model + + # sampling imposed — recorded raw on the trace, per call. + upstream_request = dialect.apply_overrides( + body, session.ctx.model, session.ctx.sampling + ) response = await session.ctx.client.get_response( dialect, body, @@ -527,12 +528,13 @@ async def _stream( dialect.error_body(f"rollout stopped: {refused}"), status=400 ) session.error = None - upstream_request = dialect.apply_overrides( - body, session.ctx.model, session.ctx.sampling - ) + upstream_request: dict | None = None started = time.time() try: turn = graph.prepare_turn(session.trace, prompt) + upstream_request = dialect.apply_overrides( + body, session.ctx.model, session.ctx.sampling + ) reply = await session.ctx.client.relay( dialect, body, @@ -618,6 +620,11 @@ async def _stream( parser_error = e except ConnectionResetError: return resp + except Exception as e: + # A mid-relay upstream failure (the provider stream died) is still a real + # exchange; record it before the error propagates to the harness. + self.record_call(session, dialect, upstream_request, started, error=e) + raise finally: producer.cancel() # Let a canceled producer enqueue EOF while unwinding. diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 2ea09013c1..c92c1d6a23 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -78,7 +78,9 @@ class ModelCall(StrictBaseModel): request: dict[str, Any] | None = None """The raw request body as sent upstream: the harness's native JSON with the rollout's model + sampling overrides applied — so it carries the effective sampling parameters - and the requested model.""" + and the requested model. The generating (renderer) client sends this conversation as + rendered token ids instead of JSON; its record keeps this native shape — the logical + exchange — just like its `response` is the completion it synthesizes.""" response: dict[str, Any] | None = None """The raw native response object (for a generating client, the completion it synthesized): provider response id, returned model, native usage. None for a failed call.""" From 970f63bbe34028cee37b7fc258119f46e8a07272 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 20:23:13 +0000 Subject: [PATCH 04/34] revert: stamp the call's wire format by route, not a dialect literal The endpoint path already identifies the format; drop DialectName and Dialect.name in favor of recording upstream_path on the call. Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/__init__.py | 3 +-- verifiers/v1/dialects/anthropic.py | 1 - verifiers/v1/dialects/base.py | 8 +------- verifiers/v1/dialects/chat.py | 1 - verifiers/v1/dialects/responses.py | 1 - verifiers/v1/interception/server.py | 2 +- verifiers/v1/trace.py | 6 +++--- 7 files changed, 6 insertions(+), 16 deletions(-) diff --git a/verifiers/v1/dialects/__init__.py b/verifiers/v1/dialects/__init__.py index 70664d27fc..72fd582f34 100644 --- a/verifiers/v1/dialects/__init__.py +++ b/verifiers/v1/dialects/__init__.py @@ -1,7 +1,7 @@ """Registered wire dialects for interception.""" from verifiers.v1.dialects.anthropic import AnthropicDialect -from verifiers.v1.dialects.base import Dialect, DialectName, StreamParser, iter_sse +from verifiers.v1.dialects.base import Dialect, StreamParser, iter_sse from verifiers.v1.dialects.chat import ( FINISH_REASONS, ChatDialect, @@ -17,7 +17,6 @@ __all__ = [ "Dialect", - "DialectName", "DIALECTS", "FINISH_REASONS", "AnthropicDialect", diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 1e1473e04b..597e076a7c 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -245,7 +245,6 @@ def finish(self) -> Response: class AnthropicDialect(Dialect[dict, AnthropicMessage]): - name = "anthropic" routes = ("/v1/messages",) aux_routes = ("/v1/messages/count_tokens",) upstream_path = "/v1/messages" diff --git a/verifiers/v1/dialects/base.py b/verifiers/v1/dialects/base.py index dfc6195045..7846978317 100644 --- a/verifiers/v1/dialects/base.py +++ b/verifiers/v1/dialects/base.py @@ -16,7 +16,7 @@ import logging from abc import ABC, abstractmethod from collections.abc import Callable, Iterator, Mapping -from typing import ClassVar, Generic, Literal, TypeVar +from typing import ClassVar, Generic, TypeVar from pydantic import BaseModel from pydantic_core import from_json @@ -26,9 +26,6 @@ ReqT = TypeVar("ReqT") RespT = TypeVar("RespT", bound=BaseModel) -DialectName = Literal["chat", "responses", "anthropic"] -"""The registered wire formats — how a trace's raw per-call `request`/`response` are shaped.""" - logger = logging.getLogger(__name__) @@ -124,9 +121,6 @@ class Dialect(ABC, Generic[ReqT, RespT]): `dialects.DIALECTS` and a harness speaking that format works end-to-end (the eval client and interception server are generic over this interface).""" - name: ClassVar[DialectName] - """This dialect's wire-format name, recorded on the trace's per-call records.""" - routes: ClassVar[tuple[str, ...]] """The endpoint path(s) a program's SDK posts model turns to. The interception server serves one handler per route, so the wire format is resolved from the route the SDK chose (it diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index 8e94d89de5..531c45df8b 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -291,7 +291,6 @@ def finish(self) -> Response: class ChatDialect(Dialect[dict, ChatCompletion]): - name = "chat" routes = ("/v1/chat/completions",) upstream_path = "/chat/completions" response_type = ModdedChatCompletion diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index fab9e53cb5..74bbaa27ca 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -279,7 +279,6 @@ def finish(self) -> Response: class ResponsesDialect(Dialect[dict, OpenAIResponse]): - name = "responses" routes = ("/v1/responses",) upstream_path = "/responses" response_type = OpenAIResponse diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 25bb70b506..5518c9e666 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -252,7 +252,7 @@ def record_call( session.trace.calls.append( ModelCall( node=node, - dialect=dialect.name, + endpoint=dialect.upstream_path, request=request, response=response, response_headers=headers, diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index c92c1d6a23..0814eb85eb 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -15,7 +15,6 @@ from verifiers.v1.judge import JudgeResponse from verifiers.v1 import graph -from verifiers.v1.dialects import DialectName from verifiers.v1.errors import ProviderError from verifiers.v1.graph import MessageNode from verifiers.v1.harness import HarnessConfig @@ -73,8 +72,9 @@ class ModelCall(StrictBaseModel): """Index into `Trace.nodes` of the assistant node this call committed — the link into the message graph (the call's conversation is that node's root-to-self path). None for a call that committed no turn (see `error`).""" - dialect: DialectName | None = None - """The wire format `request` and `response` are shaped as.""" + endpoint: str | None = None + """The provider endpoint path the request went to (e.g. `/chat/completions`) — says + which wire format `request` and `response` are in.""" request: dict[str, Any] | None = None """The raw request body as sent upstream: the harness's native JSON with the rollout's model + sampling overrides applied — so it carries the effective sampling parameters From 985f16684b2381702a68e86e32aa4f279512e67f Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 20:34:31 +0000 Subject: [PATCH 05/34] fix(v1): record the exchange when a non-stream commit fails A completed provider exchange whose turn commit raises was dropped from Trace.calls on the non-streaming path (the streaming path already recorded it). Record it with the response and the failure before propagating, matching the stream path. Co-Authored-By: Claude Fable 5 --- verifiers/v1/interception/server.py | 18 ++++++++++++++++-- verifiers/v1/trace.py | 4 +++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 5518c9e666..9313623d42 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -458,8 +458,22 @@ def serve(response: Response) -> web.Response: session.trace.id, len(response.message.tool_calls or []), ) - node = turn.commit(response, tools) # one node per new message; - # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph) + try: + node = turn.commit(response, tools) # one node per new message; + # branches fall out of walking the graph (see Trace.branches / graph) + except Exception as e: + # The provider exchange completed even though committing it failed; + # record it, with the failure, before the error propagates. + self.record_call( + session, + dialect, + upstream_request, + started, + response=response.raw, + headers=response.raw_headers, + error=e, + ) + raise self.record_call( session, dialect, diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 0814eb85eb..bc5726fbfc 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -83,7 +83,9 @@ class ModelCall(StrictBaseModel): exchange — just like its `response` is the completion it synthesizes.""" response: dict[str, Any] | None = None """The raw native response object (for a generating client, the completion it - synthesized): provider response id, returned model, native usage. None for a failed call.""" + synthesized): provider response id, returned model, native usage. None when the + exchange itself failed; kept alongside `error` when a response arrived but + recording its turn failed.""" response_headers: dict[str, str] | None = None """Provider response headers (request ids, rate limits), when the transport exposes them.""" time: TimeSpan = Field(default_factory=TimeSpan) From f00310d303215805e1963796734a04103f177956 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 20:38:44 +0000 Subject: [PATCH 06/34] fix(v1): keep response data on stream-failure call records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stream failure record now carries what the exchange already produced: provider headers on a mid-relay death, plus the assembled native payload when only the commit failed — matching the non-streaming path. Co-Authored-By: Claude Fable 5 --- verifiers/v1/interception/server.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 9313623d42..97045374ed 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -637,7 +637,14 @@ async def _stream( except Exception as e: # A mid-relay upstream failure (the provider stream died) is still a real # exchange; record it before the error propagates to the harness. - self.record_call(session, dialect, upstream_request, started, error=e) + self.record_call( + session, + dialect, + upstream_request, + started, + headers=reply.headers, + error=e, + ) raise finally: producer.cancel() @@ -647,6 +654,7 @@ async def _stream( await asyncio.gather(producer, return_exceptions=True) await reply.close() + response: Response | None = None try: if parser_error is not None: raise parser_error @@ -663,7 +671,17 @@ async def _stream( ) logger.debug("intercept stream turn: id=%s", session.trace.id) except Exception as e: - self.record_call(session, dialect, upstream_request, started, error=e) + # Keep whatever the exchange produced: the assembled payload when only the + # commit failed, and the provider headers either way. + self.record_call( + session, + dialect, + upstream_request, + started, + response=response.raw if response is not None else None, + headers=reply.headers, + error=e, + ) raise finally: # Release the withheld events only now — after the commit — then close. From 7a7f0a9aec198a6a757a6f0e4de504e1a0816d48 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 20:41:49 +0000 Subject: [PATCH 07/34] refactor: one commit-side record_call per path, in a finally Co-Authored-By: Claude Fable 5 --- verifiers/v1/interception/server.py | 45 ++++++++++++----------------- 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 97045374ed..f504758636 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -458,31 +458,27 @@ def serve(response: Response) -> web.Response: session.trace.id, len(response.message.tool_calls or []), ) + node: int | None = None + error: Exception | None = None try: node = turn.commit(response, tools) # one node per new message; # branches fall out of walking the graph (see Trace.branches / graph) except Exception as e: - # The provider exchange completed even though committing it failed; - # record it, with the failure, before the error propagates. + error = e + raise + finally: + # The exchange completed either way; a commit failure is coupled + # to the call it belongs to (node stays None). self.record_call( session, dialect, upstream_request, started, + node=node, response=response.raw, headers=response.raw_headers, - error=e, + error=error, ) - raise - self.record_call( - session, - dialect, - upstream_request, - started, - node=node, - response=response.raw, - headers=response.raw_headers, - ) # Hand back to the program when the model wants a tool (the program runs it) or # when there's no user simulator to keep the conversation going. if response.message.tool_calls or session.user is None: @@ -655,35 +651,30 @@ async def _stream( await reply.close() response: Response | None = None + node: int | None = None + error: Exception | None = None try: if parser_error is not None: raise parser_error response = parser.finish() node = turn.commit(response, tools) - self.record_call( - session, - dialect, - upstream_request, - started, - node=node, - response=response.raw, - headers=reply.headers, - ) logger.debug("intercept stream turn: id=%s", session.trace.id) except Exception as e: - # Keep whatever the exchange produced: the assembled payload when only the - # commit failed, and the provider headers either way. + error = e + raise + finally: + # Record whatever the exchange produced: the assembled payload when only + # the commit failed, the provider headers either way. self.record_call( session, dialect, upstream_request, started, + node=node, response=response.raw if response is not None else None, headers=reply.headers, - error=e, + error=error, ) - raise - finally: # Release the withheld events only now — after the commit — then close. with contextlib.suppress(ConnectionResetError): for event in deferred: From 5087cf022bed03c6d07569f52706d9e11c302ba3 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 20:47:17 +0000 Subject: [PATCH 08/34] refactor: one per-exchange record_call per interception path Wrap each path's whole exchange (overrides -> call -> commit) in a single try/finally whose finally appends the one ModelCall, instead of recording at every failure site. Error handlers just stash the error; a harness disconnect mid-stream now also records (the exchange happened), and a post-commit delivery failure keeps the success record. Co-Authored-By: Claude Fable 5 --- verifiers/v1/interception/server.py | 375 ++++++++++++++-------------- 1 file changed, 183 insertions(+), 192 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index f504758636..01f09108b6 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -394,89 +394,83 @@ def serve(response: Response) -> web.Response: turn = graph.prepare_turn(session.trace, prompt) session.error = None upstream_request: dict | None = None + call_response: Response | None = None + node: int | None = None + error: Exception | None = None started = time.time() try: - # What actually goes upstream: the native body with the rollout's model + - # sampling imposed — recorded raw on the trace, per call. - upstream_request = dialect.apply_overrides( - body, session.ctx.model, session.ctx.sampling - ) - response = await session.ctx.client.get_response( - dialect, - body, - session.ctx.model, - session.ctx.sampling, - headers=headers, - session_id=session.trace.id, - turn=turn, - ) - except OverlongPromptError as e: - # An overlong prompt is a budget limit, not a crash: end the rollout cleanly - # as a truncation — return the last turn if there is one, else refuse to halt - # the harness (same shape as `refused` above). - self.record_call( - session, dialect, upstream_request, started, error=e - ) - session.trace.stop("context_length") - logger.debug("prompt too long: id=%s", session.trace.id) - if response is None: + try: + # What actually goes upstream: the native body with the rollout's model + + # sampling imposed — recorded raw on the trace, per call. + upstream_request = dialect.apply_overrides( + body, session.ctx.model, session.ctx.sampling + ) + call_response = await session.ctx.client.get_response( + dialect, + body, + session.ctx.model, + session.ctx.sampling, + headers=headers, + session_id=session.trace.id, + turn=turn, + ) + logger.debug( + "intercept turn: id=%s tools=%d", + session.trace.id, + len(call_response.message.tool_calls or []), + ) + # One node per new message; branches fall out of walking the + # graph (see Trace.branches / verifiers.v1.graph). + node = turn.commit(call_response, tools) + response = call_response + except OverlongPromptError as e: + # An overlong prompt is a budget limit, not a crash: end the rollout cleanly + # as a truncation — return the last turn if there is one, else refuse to halt + # the harness (same shape as `refused` above). + error = e + session.trace.stop("context_length") + logger.debug("prompt too long: id=%s", session.trace.id) + if response is None: + return web.json_response( + dialect.error_body("rollout stopped: context_length"), + status=400, + ) + return serve(response) + except RolloutError as e: + # Stash the real cause; the rollout re-raises it after the harness returns. + # Relay the provider's status so the harness SDK retries 5xx/429 and not 4xx. + error = e + session.error = e + logger.warning( + "model call failed: id=%s %s: %s", + session.trace.id, + type(e).__name__, + e, + ) return web.json_response( - dialect.error_body("rollout stopped: context_length"), - status=400, + dialect.error_body(str(e)), + status=getattr(e, "status_code", 502), ) - return serve(response) - except RolloutError as e: - # Stash the real cause; the rollout re-raises it after the harness returns. - # Relay the provider's status so the harness SDK retries 5xx/429 and not 4xx. - self.record_call( - session, dialect, upstream_request, started, error=e - ) - session.error = e - logger.warning( - "model call failed: id=%s %s: %s", - session.trace.id, - type(e).__name__, - e, - ) - return web.json_response( - dialect.error_body(str(e)), - status=getattr(e, "status_code", 502), - ) - except Exception as e: # surface to the program as an API error - self.record_call( - session, dialect, upstream_request, started, error=e - ) - logger.warning( - "model call failed: id=%s %s: %s", - session.trace.id, - type(e).__name__, - e, - ) - return web.json_response(dialect.error_body(str(e)), status=502) - logger.debug( - "intercept turn: id=%s tools=%d", - session.trace.id, - len(response.message.tool_calls or []), - ) - node: int | None = None - error: Exception | None = None - try: - node = turn.commit(response, tools) # one node per new message; - # branches fall out of walking the graph (see Trace.branches / graph) - except Exception as e: - error = e - raise + except Exception as e: # surface to the program as an API error + error = e + logger.warning( + "model call failed: id=%s %s: %s", + session.trace.id, + type(e).__name__, + e, + ) + return web.json_response(dialect.error_body(str(e)), status=502) finally: - # The exchange completed either way; a commit failure is coupled - # to the call it belongs to (node stays None). + # The turn's one per-exchange record: whatever the exchange produced, + # plus the error that ended it (if any). self.record_call( session, dialect, upstream_request, started, node=node, - response=response.raw, - headers=response.raw_headers, + response=call_response.raw if call_response else None, + headers=call_response.raw_headers if call_response else None, error=error, ) # Hand back to the program when the model wants a tool (the program runs it) or @@ -539,132 +533,135 @@ async def _stream( ) session.error = None upstream_request: dict | None = None + reply = None + response: Response | None = None + node: int | None = None + error: Exception | None = None started = time.time() try: - turn = graph.prepare_turn(session.trace, prompt) - upstream_request = dialect.apply_overrides( - body, session.ctx.model, session.ctx.sampling - ) - reply = await session.ctx.client.relay( - dialect, - body, - session.ctx.model, - session.ctx.sampling, - headers=request.headers, - session_id=session.trace.id, - ) - except OverlongPromptError as e: - self.record_call(session, dialect, upstream_request, started, error=e) - session.trace.stop("context_length") - logger.debug("prompt too long: id=%s", session.trace.id) - return web.json_response( - dialect.error_body("rollout stopped: context_length"), status=400 - ) - except RolloutError as e: - self.record_call(session, dialect, upstream_request, started, error=e) - session.error = e - logger.warning( - "model call failed: id=%s %s: %s", - session.trace.id, - type(e).__name__, - e, + try: + turn = graph.prepare_turn(session.trace, prompt) + upstream_request = dialect.apply_overrides( + body, session.ctx.model, session.ctx.sampling + ) + reply = await session.ctx.client.relay( + dialect, + body, + session.ctx.model, + session.ctx.sampling, + headers=request.headers, + session_id=session.trace.id, + ) + except OverlongPromptError as e: + error = e + session.trace.stop("context_length") + logger.debug("prompt too long: id=%s", session.trace.id) + return web.json_response( + dialect.error_body("rollout stopped: context_length"), status=400 + ) + except RolloutError as e: + error = e + session.error = e + logger.warning( + "model call failed: id=%s %s: %s", + session.trace.id, + type(e).__name__, + e, + ) + return web.json_response( + dialect.error_body(str(e)), status=getattr(e, "status_code", 502) + ) + except Exception as e: # surface to the program as an API error + error = e + logger.warning("model call failed: id=%s %s", session.trace.id, e) + return web.json_response(dialect.error_body(str(e)), status=502) + resp = web.StreamResponse( + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"} ) - return web.json_response( - dialect.error_body(str(e)), status=getattr(e, "status_code", 502) + resp.content_type = reply.content_type.split(";")[0].strip() + # Parse complete events as they relay, avoiding a full-stream byte copy. + parser = dialect.stream_parser() + feed_event = parser.feed + on_done = parser.on_done + # One bounded producer avoids per-event tasks; keepalive timeouts only cancel readiness waits. + queue: asyncio.Queue[bytes | None] = asyncio.Queue( + maxsize=_STREAM_QUEUE_MAXSIZE ) - except Exception as e: # surface to the program as an API error - self.record_call(session, dialect, upstream_request, started, error=e) - logger.warning("model call failed: id=%s %s", session.trace.id, e) - return web.json_response(dialect.error_body(str(e)), status=502) - resp = web.StreamResponse( - headers={"Cache-Control": "no-cache", "Connection": "keep-alive"} - ) - resp.content_type = reply.content_type.split(";")[0].strip() - # Parse complete events as they relay, avoiding a full-stream byte copy. - parser = dialect.stream_parser() - feed_event = parser.feed - on_done = parser.on_done - # One bounded producer avoids per-event tasks; keepalive timeouts only cancel readiness waits. - queue: asyncio.Queue[bytes | None] = asyncio.Queue( - maxsize=_STREAM_QUEUE_MAXSIZE - ) - ready = asyncio.Event() - producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready)) - parser_error: Exception | None = None - # SSE events from the turn-ending one onward (the terminal event and any trailing - # `[DONE]`), withheld until the turn is committed: a client that ends its turn on the - # terminal event (e.g. codex on `response.completed`) would otherwise reach scoring - # with the turn still unrecorded. - deferred: list[bytes] = [] - try: - await resp.prepare(request) - while True: - try: - async with asyncio.timeout(_KEEPALIVE_INTERVAL_SECONDS): - await ready.wait() - except TimeoutError: - await resp.write(b": keepalive\n\n") - continue - chunk = queue.get_nowait() - if queue.empty(): - ready.clear() - if chunk is None: - await producer - break - if deferred or dialect.is_terminal_event(chunk): + ready = asyncio.Event() + producer = asyncio.create_task(_queue_chunks(reply.chunks, queue, ready)) + parser_error: Exception | None = None + # SSE events from the turn-ending one onward (the terminal event and any trailing + # `[DONE]`), withheld until the turn is committed: a client that ends its turn on the + # terminal event (e.g. codex on `response.completed`) would otherwise reach scoring + # with the turn still unrecorded. + deferred: list[bytes] = [] + try: + await resp.prepare(request) + while True: + try: + async with asyncio.timeout(_KEEPALIVE_INTERVAL_SECONDS): + await ready.wait() + except TimeoutError: + await resp.write(b": keepalive\n\n") + continue + chunk = queue.get_nowait() + if queue.empty(): + ready.clear() + if chunk is None: + await producer + break + if deferred or dialect.is_terminal_event(chunk): + if parser_error is None: + try: + if on_done is not None and is_sse_done_event(chunk): + on_done() + feed_event(chunk) + except Exception as e: + parser_error = e + # forwarded after the turn is committed, below + deferred.append(chunk) + continue + await resp.write(chunk) if parser_error is None: try: - if on_done is not None and is_sse_done_event(chunk): - on_done() feed_event(chunk) except Exception as e: parser_error = e - # forwarded after the turn is committed, below - deferred.append(chunk) - continue - await resp.write(chunk) - if parser_error is None: - try: - feed_event(chunk) - except Exception as e: - parser_error = e - except ConnectionResetError: + except ConnectionResetError as e: + # The harness went away mid-stream; the provider exchange still happened. + error = e + return resp + finally: + producer.cancel() + # Let a canceled producer enqueue EOF while unwinding. + if queue.full(): + queue.get_nowait() + await asyncio.gather(producer, return_exceptions=True) + await reply.close() + + try: + if parser_error is not None: + raise parser_error + response = parser.finish() + node = turn.commit(response, tools) + logger.debug("intercept stream turn: id=%s", session.trace.id) + finally: + # Release the withheld events only now — after the commit — then close. + with contextlib.suppress(ConnectionResetError): + for event in deferred: + await resp.write(event) + await resp.write_eof() return resp except Exception as e: - # A mid-relay upstream failure (the provider stream died) is still a real - # exchange; record it before the error propagates to the harness. - self.record_call( - session, - dialect, - upstream_request, - started, - headers=reply.headers, - error=e, - ) - raise - finally: - producer.cancel() - # Let a canceled producer enqueue EOF while unwinding. - if queue.full(): - queue.get_nowait() - await asyncio.gather(producer, return_exceptions=True) - await reply.close() - - response: Response | None = None - node: int | None = None - error: Exception | None = None - try: - if parser_error is not None: - raise parser_error - response = parser.finish() - node = turn.commit(response, tools) - logger.debug("intercept stream turn: id=%s", session.trace.id) - except Exception as e: - error = e + # Anything that propagates (a mid-relay upstream failure, a parser or commit + # error) ends a real exchange; couple it to the record unless the turn already + # committed (then only post-commit delivery failed). + if node is None: + error = e raise finally: - # Record whatever the exchange produced: the assembled payload when only - # the commit failed, the provider headers either way. + # The turn's one per-exchange record: whatever the exchange produced, plus + # the error that ended it (if any). self.record_call( session, dialect, @@ -672,15 +669,9 @@ async def _stream( started, node=node, response=response.raw if response is not None else None, - headers=reply.headers, + headers=reply.headers if reply is not None else None, error=error, ) - # Release the withheld events only now — after the commit — then close. - with contextlib.suppress(ConnectionResetError): - for event in deferred: - await resp.write(event) - await resp.write_eof() - return resp async def handle_aux( self, request: web.Request, dialect: Dialect, route: str From bed6e988d7cfddfa4cf02a73098ac333a82a8059 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 22:24:42 +0000 Subject: [PATCH 09/34] fix(v1): sound per-call records under finally-time recording Review follow-ups from the single-record refactor: tracebacks are formatted from the exception object (the finally runs after handlers exit, where format_exc sees nothing), a cancelled exchange is recorded with its CancelledError instead of as a phantom error-free call, and the streaming span now starts after prepare_turn like the non-streaming one. Co-Authored-By: Claude Fable 5 --- verifiers/v1/interception/server.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 01f09108b6..de44c4453e 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -243,7 +243,7 @@ def record_call( node: int | None = None, response: dict | None = None, headers: dict[str, str] | None = None, - error: Exception | None = None, + error: BaseException | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): the raw request as sent upstream, the raw native response, timing, and — when the @@ -263,9 +263,11 @@ def record_call( type=type(error).__name__, message=str(error), # Provider errors already carry the actionable upstream diagnostic. + # Format from the exception object: the record is written in a + # `finally`, where the ambient exception state is already cleared. traceback=None if isinstance(error, ProviderError) - else traceback.format_exc(), + else "".join(traceback.format_exception(error)), ), ) ) @@ -460,6 +462,11 @@ def serve(response: Response) -> web.Response: e, ) return web.json_response(dialect.error_body(str(e)), status=502) + except BaseException as e: + # A cancelled exchange (harness disconnect, shutdown) is still + # recorded, coupled to its cancellation. + error = e + raise finally: # The turn's one per-exchange record: whatever the exchange produced, # plus the error that ended it (if any). @@ -537,10 +544,10 @@ async def _stream( response: Response | None = None node: int | None = None error: Exception | None = None + turn = graph.prepare_turn(session.trace, prompt) started = time.time() try: try: - turn = graph.prepare_turn(session.trace, prompt) upstream_request = dialect.apply_overrides( body, session.ctx.model, session.ctx.sampling ) @@ -652,10 +659,10 @@ async def _stream( await resp.write(event) await resp.write_eof() return resp - except Exception as e: + except BaseException as e: # Anything that propagates (a mid-relay upstream failure, a parser or commit - # error) ends a real exchange; couple it to the record unless the turn already - # committed (then only post-commit delivery failed). + # error, a cancellation) ends a real exchange; couple it to the record unless + # the turn already committed (then only post-commit delivery failed). if node is None: error = e raise From cd9a5a14482ecbc7cdbcec2db30468ad24a63af7 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 23:57:42 +0000 Subject: [PATCH 10/34] feat(v1): per-call status, provider headers on failed exchanges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review asks: a failed call records the HTTP status it surfaced (ModelCall.status) and keeps the provider response headers when the failure carried an HTTP response — ProviderError now carries them from the eval client's three raise sites. Co-Authored-By: Claude Fable 5 --- verifiers/v1/clients/eval.py | 6 +++++- verifiers/v1/errors.py | 19 ++++++++++++++++--- verifiers/v1/interception/server.py | 7 ++++++- verifiers/v1/trace.py | 6 +++++- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/verifiers/v1/clients/eval.py b/verifiers/v1/clients/eval.py index bd3991fa88..06ba6c7689 100644 --- a/verifiers/v1/clients/eval.py +++ b/verifiers/v1/clients/eval.py @@ -107,6 +107,7 @@ async def get_response( raise model_error( f"malformed upstream response: {type(e).__name__}: {e}", status_code=502, + headers=dict(resp.headers), ) from e # The interception server returns this full native provider object to the program. response.raw = raw @@ -170,6 +171,7 @@ async def _request( raise model_error( f"upstream {e.response.status_code}: {e.response.text}", status_code=e.response.status_code, + headers=dict(e.response.headers), ) from e return response if response.status_code < 400: @@ -179,7 +181,9 @@ async def _request( finally: await response.aclose() raise model_error( - f"upstream {response.status_code}: {text}", status_code=response.status_code + f"upstream {response.status_code}: {text}", + status_code=response.status_code, + headers=dict(response.headers), ) async def relay( diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index 49f508b0ce..915bc889df 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -37,9 +37,18 @@ class ProviderError(RolloutError): (5xx/429/timeout) and not deterministic ones (4xx) — relayed from the provider, or chosen for a transport fault.""" - def __init__(self, message: str = "", *, status_code: int = 502) -> None: + def __init__( + self, + message: str = "", + *, + status_code: int = 502, + headers: dict[str, str] | None = None, + ) -> None: super().__init__(message) self.status_code = status_code + self.headers = headers + """Provider response headers when the failure carried an HTTP response (request + ids, rate-limit diagnostics) — surfaced on the trace's per-call records.""" class OverlongPromptError(ProviderError): @@ -121,7 +130,10 @@ def _provider_status(e: OpenAIError | str) -> int: def model_error( - e: OpenAIError | str, *, status_code: int | None = None + e: OpenAIError | str, + *, + status_code: int | None = None, + headers: dict[str, str] | None = None, ) -> ProviderError: """Map a provider failure to our error type: an overlong prompt (a budget limit the interception server turns into a clean truncation) is told apart from any other provider call failure, which @@ -131,8 +143,9 @@ def model_error( # Some SDK errors stringify empty; fall back to the type so the message is never blank. text = str(e) or (type(e).__name__ if isinstance(e, BaseException) else "") if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES): - return OverlongPromptError(text) + return OverlongPromptError(text, headers=headers) return ProviderError( text, status_code=status_code if status_code is not None else _provider_status(e), + headers=headers, ) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index de44c4453e..e23f21c80d 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -255,7 +255,12 @@ def record_call( endpoint=dialect.upstream_path, request=request, response=response, - response_headers=headers, + # A failed exchange's HTTP response still carries diagnostics (request + # ids, rate limits): fall back to the headers stashed on the error. + response_headers=headers + if headers is not None + else getattr(error, "headers", None), + status=getattr(error, "status_code", None), time=TimeSpan(start=started, end=time.time()), error=None if error is None diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index bc5726fbfc..bb00384152 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -87,7 +87,11 @@ class ModelCall(StrictBaseModel): exchange itself failed; kept alongside `error` when a response arrived but recording its turn failed.""" response_headers: dict[str, str] | None = None - """Provider response headers (request ids, rate limits), when the transport exposes them.""" + """Provider response headers (request ids, rate limits), when the transport exposes + them — kept for failed exchanges too, when the failure carried an HTTP response.""" + status: int | None = None + """The HTTP status a failed exchange surfaced (from the provider, or chosen for a + transport fault); None on success — a recorded turn implies a 2xx exchange.""" time: TimeSpan = Field(default_factory=TimeSpan) """Wall-clock span from sending the request to the fully received response.""" error: Error | None = None From 549d058fa4e8c2d38be27e5dde508ad62aab46c6 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 00:02:08 +0000 Subject: [PATCH 11/34] fix(v1): real status on overlong per-call records OverlongPromptError inherited ProviderError's 502 default, which record_call then stamped onto context-length records. Default it to the 400 the interception server surfaces, and let model_error keep the provider's real status when the failure carried one. Co-Authored-By: Claude Fable 5 --- verifiers/v1/errors.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index 915bc889df..ad66b7a69c 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -53,7 +53,18 @@ def __init__( class OverlongPromptError(ProviderError): """The prompt exceeded the model's context window — a budget limit, ended as a clean - truncation rather than recorded as an error.""" + truncation rather than recorded as an error. Defaults to a 400 (what the interception + server surfaces for it — deterministic, so an SDK never retries it); `model_error` + keeps the provider's real status when the failure carried one.""" + + def __init__( + self, + message: str = "", + *, + status_code: int = 400, + headers: dict[str, str] | None = None, + ) -> None: + super().__init__(message, status_code=status_code, headers=headers) class HarnessError(RolloutError): @@ -143,7 +154,17 @@ def model_error( # Some SDK errors stringify empty; fall back to the type so the message is never blank. text = str(e) or (type(e).__name__ if isinstance(e, BaseException) else "") if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES): - return OverlongPromptError(text, headers=headers) + from openai import APIStatusError + + # Keep the provider's real status when the failure carried one; else the class + # default (the 400 the interception server surfaces for overlong prompts). + if status_code is None and isinstance(e, APIStatusError): + status_code = e.status_code + return OverlongPromptError( + text, + **({} if status_code is None else {"status_code": status_code}), + headers=headers, + ) return ProviderError( text, status_code=status_code if status_code is not None else _provider_status(e), From 78ee3feb1b22b14797aa29af1c83a7ff92755350 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 00:04:35 +0000 Subject: [PATCH 12/34] fix(v1): derive provider headers from SDK status errors too model_error already keeps an APIStatusError's status; keep its response headers the same way, so renderer-path failures record request ids and rate-limit diagnostics like the httpx proxy's do. Co-Authored-By: Claude Fable 5 --- verifiers/v1/errors.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index ad66b7a69c..6955925686 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -151,11 +151,15 @@ def model_error( becomes a plain `ProviderError`. `status_code` is the HTTP status surfaced to the harness (whose SDK then retries 5xx/429/timeout and not 4xx); derived from an SDK error when not given. Accepts an SDK error (the renderer) or the provider's raw error body (the httpx proxy).""" + from openai import APIStatusError + # Some SDK errors stringify empty; fall back to the type so the message is never blank. text = str(e) or (type(e).__name__ if isinstance(e, BaseException) else "") + # An SDK status error carries the provider's HTTP response; keep its diagnostics + # (request ids, rate limits) when the caller didn't pass them explicitly. + if headers is None and isinstance(e, APIStatusError): + headers = dict(e.response.headers) if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES): - from openai import APIStatusError - # Keep the provider's real status when the failure carried one; else the class # default (the 400 the interception server surfaces for overlong prompts). if status_code is None and isinstance(e, APIStatusError): From 02dcb90bbf82b49973878cfd5105f77f798a8c68 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 00:10:15 +0000 Subject: [PATCH 13/34] docs: status is HTTP diagnostics, error is the failure signal Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index bb00384152..dacc24f089 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -90,8 +90,10 @@ class ModelCall(StrictBaseModel): """Provider response headers (request ids, rate limits), when the transport exposes them — kept for failed exchanges too, when the failure carried an HTTP response.""" status: int | None = None - """The HTTP status a failed exchange surfaced (from the provider, or chosen for a - transport fault); None on success — a recorded turn implies a 2xx exchange.""" + """The HTTP status a failed exchange surfaced (the provider's, or the one chosen for + a transport fault), when the failure carried one. `error` — not this — is the failure + signal: a non-HTTP failure (a commit error, a cancellation) records no status, and a + recorded turn implies a 2xx exchange.""" time: TimeSpan = Field(default_factory=TimeSpan) """Wall-clock span from sending the request to the fully received response.""" error: Error | None = None From 9ab7ef67018f8b735e3eac3e5c0bd81a0dc53b82 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 00:12:21 +0000 Subject: [PATCH 14/34] fix(v1): every failed call records its surfaced status - rename status_code to code A plain-Exception failure returned to the harness as a 502 recorded status=None; now every failed record carries the status it surfaced (the error's own, else the generic 502), so status=None means success. The v1 provider-error field is renamed status_code -> code (init kwarg, attribute, model_error parameter). Co-Authored-By: Claude Fable 5 --- verifiers/v1/clients/eval.py | 12 ++++++------ verifiers/v1/errors.py | 22 +++++++++++----------- verifiers/v1/interception/server.py | 12 +++++++----- verifiers/v1/trace.py | 7 +++---- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/verifiers/v1/clients/eval.py b/verifiers/v1/clients/eval.py index 06ba6c7689..ab39638caf 100644 --- a/verifiers/v1/clients/eval.py +++ b/verifiers/v1/clients/eval.py @@ -106,7 +106,7 @@ async def get_response( except (ValueError, ValidationError) as e: raise model_error( f"malformed upstream response: {type(e).__name__}: {e}", - status_code=502, + code=502, headers=dict(resp.headers), ) from e # The interception server returns this full native provider object to the program. @@ -156,11 +156,11 @@ async def _request( try: response = await self.http.send(request, stream=stream) except httpx.TimeoutException as e: - raise model_error(str(e), status_code=504) from e + raise model_error(str(e), code=504) from e except httpx.HTTPError as e: - raise model_error(str(e), status_code=503) from e + raise model_error(str(e), code=503) from e except ConnectionResetError as e: - raise model_error(str(e), status_code=503) from e + raise model_error(str(e), code=503) from e if not stream: try: response.raise_for_status() @@ -170,7 +170,7 @@ async def _request( # make an information-free ProviderError raise model_error( f"upstream {e.response.status_code}: {e.response.text}", - status_code=e.response.status_code, + code=e.response.status_code, headers=dict(e.response.headers), ) from e return response @@ -182,7 +182,7 @@ async def _request( await response.aclose() raise model_error( f"upstream {response.status_code}: {text}", - status_code=response.status_code, + code=response.status_code, headers=dict(response.headers), ) diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index 6955925686..4a8f62e303 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -33,7 +33,7 @@ class RolloutError(Exception): class ProviderError(RolloutError): """A model-provider call failed (transport, HTTP status, timeout, or malformed response). - `status_code` is the HTTP status surfaced to the harness so its SDK retries transient faults + `code` is the HTTP status surfaced to the harness so its SDK retries transient faults (5xx/429/timeout) and not deterministic ones (4xx) — relayed from the provider, or chosen for a transport fault.""" @@ -41,11 +41,11 @@ def __init__( self, message: str = "", *, - status_code: int = 502, + code: int = 502, headers: dict[str, str] | None = None, ) -> None: super().__init__(message) - self.status_code = status_code + self.code = code self.headers = headers """Provider response headers when the failure carried an HTTP response (request ids, rate-limit diagnostics) — surfaced on the trace's per-call records.""" @@ -61,10 +61,10 @@ def __init__( self, message: str = "", *, - status_code: int = 400, + code: int = 400, headers: dict[str, str] | None = None, ) -> None: - super().__init__(message, status_code=status_code, headers=headers) + super().__init__(message, code=code, headers=headers) class HarnessError(RolloutError): @@ -143,12 +143,12 @@ def _provider_status(e: OpenAIError | str) -> int: def model_error( e: OpenAIError | str, *, - status_code: int | None = None, + code: int | None = None, headers: dict[str, str] | None = None, ) -> ProviderError: """Map a provider failure to our error type: an overlong prompt (a budget limit the interception server turns into a clean truncation) is told apart from any other provider call failure, which - becomes a plain `ProviderError`. `status_code` is the HTTP status surfaced to the harness (whose + becomes a plain `ProviderError`. `code` is the HTTP status surfaced to the harness (whose SDK then retries 5xx/429/timeout and not 4xx); derived from an SDK error when not given. Accepts an SDK error (the renderer) or the provider's raw error body (the httpx proxy).""" from openai import APIStatusError @@ -162,15 +162,15 @@ def model_error( if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES): # Keep the provider's real status when the failure carried one; else the class # default (the 400 the interception server surfaces for overlong prompts). - if status_code is None and isinstance(e, APIStatusError): - status_code = e.status_code + if code is None and isinstance(e, APIStatusError): + code = e.status_code return OverlongPromptError( text, - **({} if status_code is None else {"status_code": status_code}), + **({} if code is None else {"code": code}), headers=headers, ) return ProviderError( text, - status_code=status_code if status_code is not None else _provider_status(e), + code=code if code is not None else _provider_status(e), headers=headers, ) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index e23f21c80d..74f49c2cf0 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -230,7 +230,7 @@ def _fail( ) return web.json_response( dialect.error_body(str(error)), - status=getattr(error, "status_code", 502), + status=getattr(error, "code", 502), ) def record_call( @@ -260,7 +260,9 @@ def record_call( response_headers=headers if headers is not None else getattr(error, "headers", None), - status=getattr(error, "status_code", None), + # Every failure surfaces an HTTP status to the harness — the error's own + # when it carries one, else the generic 502 the handlers return. + status=getattr(error, "code", 502) if error is not None else None, time=TimeSpan(start=started, end=time.time()), error=None if error is None @@ -456,7 +458,7 @@ def serve(response: Response) -> web.Response: ) return web.json_response( dialect.error_body(str(e)), - status=getattr(e, "status_code", 502), + status=getattr(e, "code", 502), ) except Exception as e: # surface to the program as an API error error = e @@ -581,7 +583,7 @@ async def _stream( e, ) return web.json_response( - dialect.error_body(str(e)), status=getattr(e, "status_code", 502) + dialect.error_body(str(e)), status=getattr(e, "code", 502) ) except Exception as e: # surface to the program as an API error error = e @@ -708,7 +710,7 @@ async def handle_aux( e, ) return web.json_response( - dialect.error_body(str(e)), status=getattr(e, "status_code", 502) + dialect.error_body(str(e)), status=getattr(e, "code", 502) ) except Exception as e: logger.warning("aux call failed: id=%s %s", session.trace.id, e) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index dacc24f089..001a5d5090 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -90,10 +90,9 @@ class ModelCall(StrictBaseModel): """Provider response headers (request ids, rate limits), when the transport exposes them — kept for failed exchanges too, when the failure carried an HTTP response.""" status: int | None = None - """The HTTP status a failed exchange surfaced (the provider's, or the one chosen for - a transport fault), when the failure carried one. `error` — not this — is the failure - signal: a non-HTTP failure (a commit error, a cancellation) records no status, and a - recorded turn implies a 2xx exchange.""" + """The HTTP status a failed exchange surfaced (the provider's, one chosen for a + transport fault, or the generic 502 for an unexpected failure); None on success — + a recorded turn implies a 2xx exchange.""" time: TimeSpan = Field(default_factory=TimeSpan) """Wall-clock span from sending the request to the fully received response.""" error: Error | None = None From e7be495e2a231eba580e10b32b4d8c5f4fd1ec8c Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 00:24:35 +0000 Subject: [PATCH 15/34] chore: export ModelCall from verifiers.v1 Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 86eba56b1c..7aa7654f38 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -105,6 +105,7 @@ Branch, Error, EvalRunInfo, + ModelCall, RunInfo, TimeSpan, Timing, @@ -172,6 +173,7 @@ "AgentInfo", "RunInfo", "EvalRunInfo", + "ModelCall", "TrainRunInfo", "VersionInfo", "State", From 4ab20a67d70e659ed899091107b9daf6cb915dfb Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:17:26 +0000 Subject: [PATCH 16/34] refactor(v1)!: type the per-call records, drop raw request/response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raw bodies grew traces quadratically in turns (83% of a 10-turn tb2 trace) and the conversation is already the linked node's ancestor path. A ModelCall is now a lean typed record: node, model, sampling (the wire request minus its payload — eval-imposed knobs plus harness-chosen extras), endpoint, finish_reason, status, time, error. Response headers, Response.raw_headers, RelayReply.headers, and ProviderError headers plumbing are gone with the raw fields. Same 10-turn trace: 146 KB -> 31 KB, calls 121 KB -> 2.5 KB. Co-Authored-By: Claude Fable 5 --- tests/v1/test_e2e.py | 6 ++-- verifiers/v1/clients/client.py | 2 -- verifiers/v1/clients/eval.py | 8 +---- verifiers/v1/dialects/anthropic.py | 7 ++-- verifiers/v1/dialects/base.py | 5 +++ verifiers/v1/dialects/chat.py | 7 ++-- verifiers/v1/dialects/responses.py | 15 ++++++-- verifiers/v1/errors.py | 44 ++++-------------------- verifiers/v1/interception/server.py | 53 ++++++++++++++++++----------- verifiers/v1/trace.py | 39 +++++++++++---------- verifiers/v1/types.py | 6 +--- 11 files changed, 87 insertions(+), 105 deletions(-) diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index eda519951c..5066877d29 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -18,14 +18,12 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path): assert trace.errors == [] assert trace.num_turns == 1 assert trace.reward == 1.0 - # Every sampled turn has one raw per-call record, linked to its assistant node. + # Every sampled turn has one per-call record, linked to its assistant node. sampled = [i for i, n in enumerate(trace.nodes) if n.sampled] assert [c.node for c in trace.calls if c.error is None] == sampled for call in trace.calls: - assert call.request is not None + assert call.model and call.sampling is not None assert call.time.duration > 0 - if call.error is None: - assert call.response is not None @pytest.mark.e2e diff --git a/verifiers/v1/clients/client.py b/verifiers/v1/clients/client.py index 70bf59a6ad..04b6adbfb3 100644 --- a/verifiers/v1/clients/client.py +++ b/verifiers/v1/clients/client.py @@ -26,8 +26,6 @@ class RelayReply: content_type: str chunks: AsyncIterator[bytes] close: Callable[[], Awaitable[None]] - headers: dict[str, str] | None = None - """Provider response headers, for the trace's per-call records.""" class Client(ABC): diff --git a/verifiers/v1/clients/eval.py b/verifiers/v1/clients/eval.py index ab39638caf..452d084621 100644 --- a/verifiers/v1/clients/eval.py +++ b/verifiers/v1/clients/eval.py @@ -107,11 +107,9 @@ async def get_response( raise model_error( f"malformed upstream response: {type(e).__name__}: {e}", code=502, - headers=dict(resp.headers), ) from e # The interception server returns this full native provider object to the program. response.raw = raw - response.raw_headers = dict(resp.headers) return response def _headers( @@ -171,7 +169,6 @@ async def _request( raise model_error( f"upstream {e.response.status_code}: {e.response.text}", code=e.response.status_code, - headers=dict(e.response.headers), ) from e return response if response.status_code < 400: @@ -181,9 +178,7 @@ async def _request( finally: await response.aclose() raise model_error( - f"upstream {response.status_code}: {text}", - code=response.status_code, - headers=dict(response.headers), + f"upstream {response.status_code}: {text}", code=response.status_code ) async def relay( @@ -222,7 +217,6 @@ async def chunks(): content_type=resp.headers.get("content-type", "text/event-stream"), chunks=chunks(), close=resp.aclose, - headers=dict(resp.headers), ) async def relay_aux( diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 597e076a7c..33b9400e05 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -239,12 +239,13 @@ def finish(self) -> Response: for index, parts in self.partial_json.items(): self.blocks[index]["input"] = json.loads("".join(parts) or "{}") self.message["content"] = [self.blocks[index] for index in sorted(self.blocks)] - response = response_from_wire(self.validate_response(self.message)) - response.raw = self.message - return response + return response_from_wire(self.validate_response(self.message)) class AnthropicDialect(Dialect[dict, AnthropicMessage]): + payload_fields = frozenset( + {"messages", "system", "tools", "model", "stream", "stream_options"} + ) routes = ("/v1/messages",) aux_routes = ("/v1/messages/count_tokens",) upstream_path = "/v1/messages" diff --git a/verifiers/v1/dialects/base.py b/verifiers/v1/dialects/base.py index 7846978317..83d258ba18 100644 --- a/verifiers/v1/dialects/base.py +++ b/verifiers/v1/dialects/base.py @@ -121,6 +121,11 @@ class Dialect(ABC, Generic[ReqT, RespT]): `dialects.DIALECTS` and a harness speaking that format works end-to-end (the eval client and interception server are generic over this interface).""" + payload_fields: ClassVar[frozenset[str]] = frozenset() + """Request keys that carry the payload (conversation, tools, model) or transport + framing rather than settings — stripped when recording a call's effective settings + (`ModelCall.sampling`) on the trace.""" + routes: ClassVar[tuple[str, ...]] """The endpoint path(s) a program's SDK posts model turns to. The interception server serves one handler per route, so the wire format is resolved from the route the SDK chose (it diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index 531c45df8b..1922e48401 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -285,12 +285,13 @@ def finish(self) -> Response: ], "usage": self.usage, } - response = response_from_wire(ModdedChatCompletion.model_validate(completion)) - response.raw = completion - return response + return response_from_wire(ModdedChatCompletion.model_validate(completion)) class ChatDialect(Dialect[dict, ChatCompletion]): + payload_fields = frozenset( + {"messages", "tools", "model", "stream", "stream_options"} + ) routes = ("/v1/chat/completions",) upstream_path = "/chat/completions" response_type = ModdedChatCompletion diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index 74bbaa27ca..081b4a9fe1 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -270,15 +270,24 @@ def finish(self) -> Response: events = self.terminal_events or self.events for event in iter_sse_reverse(b"".join(events)): if event.get("type") in FINAL_EVENTS: - response = response_from_wire( + return response_from_wire( OpenAIResponse.model_validate(event["response"]) ) - response.raw = event["response"] - return response raise ValueError("Responses stream ended without a terminal event") class ResponsesDialect(Dialect[dict, OpenAIResponse]): + payload_fields = frozenset( + { + "input", + "instructions", + "prompt", + "tools", + "model", + "stream", + "stream_options", + } + ) routes = ("/v1/responses",) upstream_path = "/responses" response_type = OpenAIResponse diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index 4a8f62e303..bf62d7f472 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -37,18 +37,9 @@ class ProviderError(RolloutError): (5xx/429/timeout) and not deterministic ones (4xx) — relayed from the provider, or chosen for a transport fault.""" - def __init__( - self, - message: str = "", - *, - code: int = 502, - headers: dict[str, str] | None = None, - ) -> None: + def __init__(self, message: str = "", *, code: int = 502) -> None: super().__init__(message) self.code = code - self.headers = headers - """Provider response headers when the failure carried an HTTP response (request - ids, rate-limit diagnostics) — surfaced on the trace's per-call records.""" class OverlongPromptError(ProviderError): @@ -57,14 +48,8 @@ class OverlongPromptError(ProviderError): server surfaces for it — deterministic, so an SDK never retries it); `model_error` keeps the provider's real status when the failure carried one.""" - def __init__( - self, - message: str = "", - *, - code: int = 400, - headers: dict[str, str] | None = None, - ) -> None: - super().__init__(message, code=code, headers=headers) + def __init__(self, message: str = "", *, code: int = 400) -> None: + super().__init__(message, code=code) class HarnessError(RolloutError): @@ -140,12 +125,7 @@ def _provider_status(e: OpenAIError | str) -> int: return 502 -def model_error( - e: OpenAIError | str, - *, - code: int | None = None, - headers: dict[str, str] | None = None, -) -> ProviderError: +def model_error(e: OpenAIError | str, *, code: int | None = None) -> ProviderError: """Map a provider failure to our error type: an overlong prompt (a budget limit the interception server turns into a clean truncation) is told apart from any other provider call failure, which becomes a plain `ProviderError`. `code` is the HTTP status surfaced to the harness (whose @@ -155,22 +135,10 @@ def model_error( # Some SDK errors stringify empty; fall back to the type so the message is never blank. text = str(e) or (type(e).__name__ if isinstance(e, BaseException) else "") - # An SDK status error carries the provider's HTTP response; keep its diagnostics - # (request ids, rate limits) when the caller didn't pass them explicitly. - if headers is None and isinstance(e, APIStatusError): - headers = dict(e.response.headers) if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES): # Keep the provider's real status when the failure carried one; else the class # default (the 400 the interception server surfaces for overlong prompts). if code is None and isinstance(e, APIStatusError): code = e.status_code - return OverlongPromptError( - text, - **({} if code is None else {"code": code}), - headers=headers, - ) - return ProviderError( - text, - code=code if code is not None else _provider_status(e), - headers=headers, - ) + return OverlongPromptError(text, **({} if code is None else {"code": code})) + return ProviderError(text, code=code if code is not None else _provider_status(e)) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 74f49c2cf0..fa72863514 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -54,7 +54,7 @@ ) from verifiers.v1.session import RolloutSession from verifiers.v1.trace import Error, ModelCall, TimeSpan -from verifiers.v1.types import Messages, Response, Tool +from verifiers.v1.types import FinishReason, Messages, Response, SamplingConfig, Tool logger = logging.getLogger(__name__) @@ -241,25 +241,38 @@ def record_call( started: float, *, node: int | None = None, - response: dict | None = None, - headers: dict[str, str] | None = None, + finish_reason: "FinishReason" = None, error: BaseException | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): - the raw request as sent upstream, the raw native response, timing, and — when the - call committed no turn — the error, coupled to the exchange that raised it. Called + the model + effective settings that went upstream, timing, and — when the call + committed no turn — the error, coupled to the exchange that raised it. Called once per real exchange; replayed/coalesced SDK retries never reach it.""" + sampling = None + if request is not None: + # The wire request minus its payload is the call's effective settings: the + # eval-imposed knobs plus whatever the harness set that the eval left alone. + try: + sampling = SamplingConfig.model_validate( + { + k: v + for k, v in request.items() + if k not in dialect.payload_fields + } + ) + except ValidationError: + # A malformed harness knob must not kill recording (this runs in the + # exchange's `finally`); the provider rejects the request on its own. + logger.warning( + "unrecordable call settings: id=%s", session.trace.id, exc_info=True + ) session.trace.calls.append( ModelCall( node=node, + model=request.get("model") if request is not None else None, + sampling=sampling, endpoint=dialect.upstream_path, - request=request, - response=response, - # A failed exchange's HTTP response still carries diagnostics (request - # ids, rate limits): fall back to the headers stashed on the error. - response_headers=headers - if headers is not None - else getattr(error, "headers", None), + finish_reason=finish_reason, # Every failure surfaces an HTTP status to the harness — the error's own # when it carries one, else the generic 502 the handlers return. status=getattr(error, "code", 502) if error is not None else None, @@ -475,16 +488,17 @@ def serve(response: Response) -> web.Response: error = e raise finally: - # The turn's one per-exchange record: whatever the exchange produced, - # plus the error that ended it (if any). + # The turn's one per-exchange record: settings, timing, outcome, and + # the error that ended it (if any). self.record_call( session, dialect, upstream_request, started, node=node, - response=call_response.raw if call_response else None, - headers=call_response.raw_headers if call_response else None, + finish_reason=call_response.finish_reason + if call_response + else None, error=error, ) # Hand back to the program when the model wants a tool (the program runs it) or @@ -674,16 +688,15 @@ async def _stream( error = e raise finally: - # The turn's one per-exchange record: whatever the exchange produced, plus - # the error that ended it (if any). + # The turn's one per-exchange record: settings, timing, outcome, and the + # error that ended it (if any). self.record_call( session, dialect, upstream_request, started, node=node, - response=response.raw if response is not None else None, - headers=reply.headers if reply is not None else None, + finish_reason=response.finish_reason if response is not None else None, error=error, ) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 001a5d5090..540cd04634 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -23,6 +23,7 @@ from verifiers.v1.task import DataT, WireTaskData from verifiers.v1.types import ( AssistantMessage, + FinishReason, KeptTokens, Messages, SamplingConfig, @@ -63,32 +64,30 @@ class Error(StrictBaseModel): class ModelCall(StrictBaseModel): - """One provider exchange behind a sampled turn, kept raw: the request as sent upstream - and the native response, untyped. Recorded by the interception server for every real - exchange — an SDK-level retry that replays or coalesces onto an earlier attempt adds - nothing, a failed attempt is recorded with its `error`.""" + """One provider exchange behind a sampled turn. Recorded by the interception server + for every real exchange — an SDK-level retry that replays or coalesces onto an + earlier attempt adds nothing, a failed attempt is recorded with its `error`. The + conversation itself is not repeated here: it is the linked node's root-to-self path + in the message graph.""" node: int | None = None """Index into `Trace.nodes` of the assistant node this call committed — the link into the message graph (the call's conversation is that node's root-to-self path). None for a call that committed no turn (see `error`).""" + model: str | None = None + """The model requested from the provider. The rollout's model override makes this + `agent.model` on every call; recorded per call because it is cheap and provable.""" + sampling: SamplingConfig | None = None + """The effective per-call request settings: everything on the wire request except its + payload (conversation, tools, model) — the eval-imposed knobs plus whatever the + harness set that the eval left alone (`seed`, `stop`, `tool_choice`, + `response_format`, ... ride along as extras).""" endpoint: str | None = None """The provider endpoint path the request went to (e.g. `/chat/completions`) — says - which wire format `request` and `response` are in.""" - request: dict[str, Any] | None = None - """The raw request body as sent upstream: the harness's native JSON with the rollout's - model + sampling overrides applied — so it carries the effective sampling parameters - and the requested model. The generating (renderer) client sends this conversation as - rendered token ids instead of JSON; its record keeps this native shape — the logical - exchange — just like its `response` is the completion it synthesizes.""" - response: dict[str, Any] | None = None - """The raw native response object (for a generating client, the completion it - synthesized): provider response id, returned model, native usage. None when the - exchange itself failed; kept alongside `error` when a response arrived but - recording its turn failed.""" - response_headers: dict[str, str] | None = None - """Provider response headers (request ids, rate limits), when the transport exposes - them — kept for failed exchanges too, when the failure carried an HTTP response.""" + which wire dialect the exchange spoke.""" + finish_reason: FinishReason = None + """Why the model stopped, normalized (`stop` / `length` / `tool_calls`); None for a + failed call or an unrecognized provider reason.""" status: int | None = None """The HTTP status a failed exchange surfaced (the provider's, one chosen for a transport fault, or the generic 502 for an unexpected failure); None on success — @@ -97,7 +96,7 @@ class ModelCall(StrictBaseModel): """Wall-clock span from sending the request to the fully received response.""" error: Error | None = None """The failure that ended this call, coupled to the exchange that caused it; None on - success. A failed call still records the request it sent.""" + success. A failed call still records the settings it was sent with.""" class Branch(StrictBaseModel): diff --git a/verifiers/v1/types.py b/verifiers/v1/types.py index a3b62712cc..7ee25f8502 100644 --- a/verifiers/v1/types.py +++ b/verifiers/v1/types.py @@ -229,11 +229,7 @@ class Response(StrictBaseModel): usage: Usage | None = None tokens: TurnTokens | None = None raw: dict | None = Field(default=None, exclude=True, repr=False) - """Full native response object returned to the program; recorded raw on the trace's - per-call records (`Trace.calls`), never dumped as part of the typed response.""" - raw_headers: dict[str, str] | None = Field(default=None, exclude=True, repr=False) - """Provider response headers (request ids, rate limits), when the transport exposes - them; carried to the trace's per-call records like `raw`.""" + """Full native response object returned to the program; excluded from traces.""" class SamplingConfig(BaseModel): From 3dae631b8616becdc161d5de56bf77f3cce8e782 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:20:57 +0000 Subject: [PATCH 17/34] fix(v1): conversation-state ids are payload, not settings previous_response_id / conversation on a Responses request link to prior turns; exclude them from the per-call sampling record. Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/responses.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index 081b4a9fe1..578d5419b3 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -286,6 +286,9 @@ class ResponsesDialect(Dialect[dict, OpenAIResponse]): "model", "stream", "stream_options", + # Server-side conversation state, not settings: ids linking to prior turns. + "previous_response_id", + "conversation", } ) routes = ("/v1/responses",) From f4b42d4d3f9e08b3c8cad421a4bc01c7d216dafa Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:23:36 +0000 Subject: [PATCH 18/34] refactor(v1): whitelist the per-call sampling capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A per-dialect sampling_fields whitelist replaces the payload blacklist: payload, conversation state, and tracking fields can never leak into the record by omission — an unlisted knob is simply not recorded. Also lean the ModelCall docstrings. Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/anthropic.py | 13 +++++++++++-- verifiers/v1/dialects/base.py | 9 +++++---- verifiers/v1/dialects/chat.py | 24 ++++++++++++++++++++++-- verifiers/v1/dialects/responses.py | 22 +++++++++++----------- verifiers/v1/interception/server.py | 10 +++------- verifiers/v1/trace.py | 14 +++++--------- 6 files changed, 57 insertions(+), 35 deletions(-) diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 33b9400e05..2debbca93c 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -243,8 +243,17 @@ def finish(self) -> Response: class AnthropicDialect(Dialect[dict, AnthropicMessage]): - payload_fields = frozenset( - {"messages", "system", "tools", "model", "stream", "stream_options"} + sampling_fields = frozenset( + { + "temperature", + "top_p", + "top_k", + "max_tokens", + "stop_sequences", + "thinking", + "tool_choice", + "output_config", + } ) routes = ("/v1/messages",) aux_routes = ("/v1/messages/count_tokens",) diff --git a/verifiers/v1/dialects/base.py b/verifiers/v1/dialects/base.py index 83d258ba18..5a23887262 100644 --- a/verifiers/v1/dialects/base.py +++ b/verifiers/v1/dialects/base.py @@ -121,10 +121,11 @@ class Dialect(ABC, Generic[ReqT, RespT]): `dialects.DIALECTS` and a harness speaking that format works end-to-end (the eval client and interception server are generic over this interface).""" - payload_fields: ClassVar[frozenset[str]] = frozenset() - """Request keys that carry the payload (conversation, tools, model) or transport - framing rather than settings — stripped when recording a call's effective settings - (`ModelCall.sampling`) on the trace.""" + sampling_fields: ClassVar[frozenset[str]] = frozenset() + """Request keys that are call settings (decoding knobs, tool choice, output format) — + the whitelist scraped into the trace's per-call `ModelCall.sampling`. A whitelist so + payload, conversation state, and tracking fields can never leak into the record by + omission; an unlisted knob is simply not recorded.""" routes: ClassVar[tuple[str, ...]] """The endpoint path(s) a program's SDK posts model turns to. The interception server serves diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index 1922e48401..fce693ca23 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -289,8 +289,28 @@ def finish(self) -> Response: class ChatDialect(Dialect[dict, ChatCompletion]): - payload_fields = frozenset( - {"messages", "tools", "model", "stream", "stream_options"} + sampling_fields = frozenset( + { + "temperature", + "top_p", + "top_k", + "min_p", + "max_tokens", + "max_completion_tokens", + "reasoning_effort", + "seed", + "stop", + "n", + "logprobs", + "top_logprobs", + "logit_bias", + "frequency_penalty", + "presence_penalty", + "repetition_penalty", + "response_format", + "tool_choice", + "parallel_tool_calls", + } ) routes = ("/v1/chat/completions",) upstream_path = "/chat/completions" diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index 578d5419b3..8c3d66ac7d 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -277,18 +277,18 @@ def finish(self) -> Response: class ResponsesDialect(Dialect[dict, OpenAIResponse]): - payload_fields = frozenset( + sampling_fields = frozenset( { - "input", - "instructions", - "prompt", - "tools", - "model", - "stream", - "stream_options", - # Server-side conversation state, not settings: ids linking to prior turns. - "previous_response_id", - "conversation", + "temperature", + "top_p", + "max_output_tokens", + "max_tool_calls", + "reasoning", + "text", + "tool_choice", + "parallel_tool_calls", + "top_logprobs", + "truncation", } ) routes = ("/v1/responses",) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index fa72863514..11fbbab68a 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -250,15 +250,11 @@ def record_call( once per real exchange; replayed/coalesced SDK retries never reach it.""" sampling = None if request is not None: - # The wire request minus its payload is the call's effective settings: the - # eval-imposed knobs plus whatever the harness set that the eval left alone. + # The dialect's whitelisted settings off the wire request: the eval-imposed + # knobs plus whatever the harness set that the eval left alone. try: sampling = SamplingConfig.model_validate( - { - k: v - for k, v in request.items() - if k not in dialect.payload_fields - } + {k: v for k, v in request.items() if k in dialect.sampling_fields} ) except ValidationError: # A malformed harness knob must not kill recording (this runs in the diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 540cd04634..83bff938ff 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -64,11 +64,8 @@ class Error(StrictBaseModel): class ModelCall(StrictBaseModel): - """One provider exchange behind a sampled turn. Recorded by the interception server - for every real exchange — an SDK-level retry that replays or coalesces onto an - earlier attempt adds nothing, a failed attempt is recorded with its `error`. The - conversation itself is not repeated here: it is the linked node's root-to-self path - in the message graph.""" + """One provider exchange behind a sampled turn; its conversation is the linked + node's root-to-self path, never repeated here.""" node: int | None = None """Index into `Trace.nodes` of the assistant node this call committed — the link into @@ -78,10 +75,9 @@ class ModelCall(StrictBaseModel): """The model requested from the provider. The rollout's model override makes this `agent.model` on every call; recorded per call because it is cheap and provable.""" sampling: SamplingConfig | None = None - """The effective per-call request settings: everything on the wire request except its - payload (conversation, tools, model) — the eval-imposed knobs plus whatever the - harness set that the eval left alone (`seed`, `stop`, `tool_choice`, - `response_format`, ... ride along as extras).""" + """The call's effective settings, scraped off the wire request by the dialect's + `sampling_fields` whitelist — the eval-imposed knobs plus whatever the harness set + that the eval left alone (`seed`, `tool_choice`, `response_format`, ... as extras).""" endpoint: str | None = None """The provider endpoint path the request went to (e.g. `/chat/completions`) — says which wire dialect the exchange spoke.""" From 69f6b710486c2eec783c795999e24a768973ffc3 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:25:17 +0000 Subject: [PATCH 19/34] refactor(v1): dialects own the per-call sampling translation parse_sampling joins parse_request/parse_response on the Dialect: whitelist via sampling_fields, native aliases mapped onto the canonical SamplingConfig knobs (max_output_tokens -> max_tokens, reasoning.effort -> reasoning_effort). Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/base.py | 16 ++++++++++++---- verifiers/v1/dialects/responses.py | 9 +++++++++ verifiers/v1/interception/server.py | 8 ++------ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/verifiers/v1/dialects/base.py b/verifiers/v1/dialects/base.py index 5a23887262..82ebe7139b 100644 --- a/verifiers/v1/dialects/base.py +++ b/verifiers/v1/dialects/base.py @@ -122,10 +122,10 @@ class Dialect(ABC, Generic[ReqT, RespT]): interception server are generic over this interface).""" sampling_fields: ClassVar[frozenset[str]] = frozenset() - """Request keys that are call settings (decoding knobs, tool choice, output format) — - the whitelist scraped into the trace's per-call `ModelCall.sampling`. A whitelist so - payload, conversation state, and tracking fields can never leak into the record by - omission; an unlisted knob is simply not recorded.""" + """Request keys that are call settings — what shapes generation given the same + conversation: decoding knobs, budgets/stops, reasoning effort, output contract. + A whitelist, so payload, conversation state, and tracking fields can never leak + into the per-call record by omission; an unlisted knob is simply not recorded.""" routes: ClassVar[tuple[str, ...]] """The endpoint path(s) a program's SDK posts model turns to. The interception server serves @@ -173,6 +173,14 @@ def error_body(self, message: str) -> dict: def parse_request(self, body: ReqT) -> tuple[Messages, list[Tool] | None]: """The native request -> vf prompt + tools (for the trace).""" + def parse_sampling(self, body: ReqT) -> SamplingConfig: + """The native request's call settings -> the canonical `SamplingConfig` (for the + trace's per-call records): the `sampling_fields` whitelist, with this format's + aliases mapped onto the typed knobs; dialect-specific keys ride as extras.""" + return SamplingConfig.model_validate( + {k: v for k, v in body.items() if k in self.sampling_fields} + ) + @abstractmethod def parse_response(self, response: RespT) -> Response: """A native (non-streamed) response -> the vf `Response` we consume.""" diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index 8c3d66ac7d..91b7e41629 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -300,6 +300,15 @@ def is_terminal_event(self, chunk: bytes) -> bool: # trailing `[DONE]`, so the turn-ending event is the final event, not the sentinel. return any(marker in chunk for marker in _TERMINAL_MARKERS) + def parse_sampling(self, body: dict) -> SamplingConfig: + settings = {k: v for k, v in body.items() if k in self.sampling_fields} + if isinstance(effort := (settings.pop("reasoning", None) or {}), dict): + if effort.get("effort"): + settings["reasoning_effort"] = effort["effort"] + if "max_output_tokens" in settings: + settings["max_tokens"] = settings.pop("max_output_tokens") + return SamplingConfig.model_validate(settings) + def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]: prompt: Messages = [] if instructions := body.get("instructions"): diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 11fbbab68a..cbfb53ee53 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -54,7 +54,7 @@ ) from verifiers.v1.session import RolloutSession from verifiers.v1.trace import Error, ModelCall, TimeSpan -from verifiers.v1.types import FinishReason, Messages, Response, SamplingConfig, Tool +from verifiers.v1.types import FinishReason, Messages, Response, Tool logger = logging.getLogger(__name__) @@ -250,12 +250,8 @@ def record_call( once per real exchange; replayed/coalesced SDK retries never reach it.""" sampling = None if request is not None: - # The dialect's whitelisted settings off the wire request: the eval-imposed - # knobs plus whatever the harness set that the eval left alone. try: - sampling = SamplingConfig.model_validate( - {k: v for k, v in request.items() if k in dialect.sampling_fields} - ) + sampling = dialect.parse_sampling(request) except ValidationError: # A malformed harness knob must not kill recording (this runs in the # exchange's `finally`); the provider rejects the request on its own. From b1026cae723ecd16bd821a780c0be011e18dbcde Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:28:44 +0000 Subject: [PATCH 20/34] chore: parse_sampling speaks Sampling, not SamplingConfig Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/base.py | 8 ++++---- verifiers/v1/dialects/responses.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/verifiers/v1/dialects/base.py b/verifiers/v1/dialects/base.py index 82ebe7139b..b963d8c9b5 100644 --- a/verifiers/v1/dialects/base.py +++ b/verifiers/v1/dialects/base.py @@ -21,7 +21,7 @@ from pydantic import BaseModel from pydantic_core import from_json -from verifiers.v1.types import Messages, Response, SamplingConfig, Tool +from verifiers.v1.types import Messages, Response, Sampling, SamplingConfig, Tool ReqT = TypeVar("ReqT") RespT = TypeVar("RespT", bound=BaseModel) @@ -173,11 +173,11 @@ def error_body(self, message: str) -> dict: def parse_request(self, body: ReqT) -> tuple[Messages, list[Tool] | None]: """The native request -> vf prompt + tools (for the trace).""" - def parse_sampling(self, body: ReqT) -> SamplingConfig: - """The native request's call settings -> the canonical `SamplingConfig` (for the + def parse_sampling(self, body: ReqT) -> Sampling: + """The native request's call settings -> the canonical `Sampling` (for the trace's per-call records): the `sampling_fields` whitelist, with this format's aliases mapped onto the typed knobs; dialect-specific keys ride as extras.""" - return SamplingConfig.model_validate( + return Sampling.model_validate( {k: v for k, v in body.items() if k in self.sampling_fields} ) diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index 91b7e41629..5d770b9ec9 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -32,6 +32,7 @@ ImageUrlSource, Messages, Response, + Sampling, SamplingConfig, SystemMessage, TextContentPart, @@ -300,14 +301,14 @@ def is_terminal_event(self, chunk: bytes) -> bool: # trailing `[DONE]`, so the turn-ending event is the final event, not the sentinel. return any(marker in chunk for marker in _TERMINAL_MARKERS) - def parse_sampling(self, body: dict) -> SamplingConfig: + def parse_sampling(self, body: dict) -> Sampling: settings = {k: v for k, v in body.items() if k in self.sampling_fields} if isinstance(effort := (settings.pop("reasoning", None) or {}), dict): if effort.get("effort"): settings["reasoning_effort"] = effort["effort"] if "max_output_tokens" in settings: settings["max_tokens"] = settings.pop("max_output_tokens") - return SamplingConfig.model_validate(settings) + return Sampling.model_validate(settings) def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]: prompt: Messages = [] From 432c5c4977db297fa24d7a06ca1f577be37306bd Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:30:27 +0000 Subject: [PATCH 21/34] fix(v1): complete the reasoning-effort reverse mappings Anthropic lifts output_config.effort onto the typed knob (where apply_overrides puts the eval's reasoning effort), and Responses keeps non-effort reasoning keys (e.g. summary) instead of dropping them. Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/anthropic.py | 15 +++++++++++++++ verifiers/v1/dialects/responses.py | 13 ++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index 2debbca93c..bf324f1aa2 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -21,6 +21,7 @@ ImageUrlSource, Messages, Response, + Sampling, SamplingConfig, SystemMessage, TextContentPart, @@ -299,6 +300,20 @@ def validate_response(self, raw: dict) -> AnthropicMessage: def stream_parser(self) -> StreamParser: return AnthropicStreamParser(self.validate_response) + def parse_sampling(self, body: dict) -> Sampling: + settings = {k: v for k, v in body.items() if k in self.sampling_fields} + # Lift `output_config.effort` (where `apply_overrides` puts the eval's + # reasoning effort) onto the typed knob; keep any other output-config keys. + if isinstance(config := settings.get("output_config"), dict): + config = dict(config) + if config.get("effort"): + settings["reasoning_effort"] = config.pop("effort") + if config: + settings["output_config"] = config + else: + settings.pop("output_config") + return Sampling.model_validate(settings) + def apply_overrides(self, body: dict, model: str, sampling: SamplingConfig) -> dict: # Preserve native fields except the eval's model + sampling. `temperature`/`top_p` are # authoritative (always dropped, the eval's applied if set); `max_tokens` is required by diff --git a/verifiers/v1/dialects/responses.py b/verifiers/v1/dialects/responses.py index 5d770b9ec9..e7b016ba56 100644 --- a/verifiers/v1/dialects/responses.py +++ b/verifiers/v1/dialects/responses.py @@ -303,9 +303,16 @@ def is_terminal_event(self, chunk: bytes) -> bool: def parse_sampling(self, body: dict) -> Sampling: settings = {k: v for k, v in body.items() if k in self.sampling_fields} - if isinstance(effort := (settings.pop("reasoning", None) or {}), dict): - if effort.get("effort"): - settings["reasoning_effort"] = effort["effort"] + # Lift `reasoning.effort` onto the typed knob; keep any other reasoning keys + # (e.g. `summary`) as the wire sent them. + if isinstance(reasoning := settings.get("reasoning"), dict): + reasoning = dict(reasoning) + if reasoning.get("effort"): + settings["reasoning_effort"] = reasoning.pop("effort") + if reasoning: + settings["reasoning"] = reasoning + else: + settings.pop("reasoning") if "max_output_tokens" in settings: settings["max_tokens"] = settings.pop("max_output_tokens") return Sampling.model_validate(settings) From 12f413207bb553354f8b4c16736e685b709749bc Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:31:35 +0000 Subject: [PATCH 22/34] revert: keep ProviderError.status_code, no rename break Co-Authored-By: Claude Fable 5 --- verifiers/v1/clients/eval.py | 12 ++++++------ verifiers/v1/errors.py | 29 ++++++++++++++++++----------- verifiers/v1/interception/server.py | 12 +++++++----- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/verifiers/v1/clients/eval.py b/verifiers/v1/clients/eval.py index 452d084621..6a7df3369c 100644 --- a/verifiers/v1/clients/eval.py +++ b/verifiers/v1/clients/eval.py @@ -106,7 +106,7 @@ async def get_response( except (ValueError, ValidationError) as e: raise model_error( f"malformed upstream response: {type(e).__name__}: {e}", - code=502, + status_code=502, ) from e # The interception server returns this full native provider object to the program. response.raw = raw @@ -154,11 +154,11 @@ async def _request( try: response = await self.http.send(request, stream=stream) except httpx.TimeoutException as e: - raise model_error(str(e), code=504) from e + raise model_error(str(e), status_code=504) from e except httpx.HTTPError as e: - raise model_error(str(e), code=503) from e + raise model_error(str(e), status_code=503) from e except ConnectionResetError as e: - raise model_error(str(e), code=503) from e + raise model_error(str(e), status_code=503) from e if not stream: try: response.raise_for_status() @@ -168,7 +168,7 @@ async def _request( # make an information-free ProviderError raise model_error( f"upstream {e.response.status_code}: {e.response.text}", - code=e.response.status_code, + status_code=e.response.status_code, ) from e return response if response.status_code < 400: @@ -178,7 +178,7 @@ async def _request( finally: await response.aclose() raise model_error( - f"upstream {response.status_code}: {text}", code=response.status_code + f"upstream {response.status_code}: {text}", status_code=response.status_code ) async def relay( diff --git a/verifiers/v1/errors.py b/verifiers/v1/errors.py index bf62d7f472..b3f721bac7 100644 --- a/verifiers/v1/errors.py +++ b/verifiers/v1/errors.py @@ -33,13 +33,13 @@ class RolloutError(Exception): class ProviderError(RolloutError): """A model-provider call failed (transport, HTTP status, timeout, or malformed response). - `code` is the HTTP status surfaced to the harness so its SDK retries transient faults + `status_code` is the HTTP status surfaced to the harness so its SDK retries transient faults (5xx/429/timeout) and not deterministic ones (4xx) — relayed from the provider, or chosen for a transport fault.""" - def __init__(self, message: str = "", *, code: int = 502) -> None: + def __init__(self, message: str = "", *, status_code: int = 502) -> None: super().__init__(message) - self.code = code + self.status_code = status_code class OverlongPromptError(ProviderError): @@ -48,8 +48,8 @@ class OverlongPromptError(ProviderError): server surfaces for it — deterministic, so an SDK never retries it); `model_error` keeps the provider's real status when the failure carried one.""" - def __init__(self, message: str = "", *, code: int = 400) -> None: - super().__init__(message, code=code) + def __init__(self, message: str = "", *, status_code: int = 400) -> None: + super().__init__(message, status_code=status_code) class HarnessError(RolloutError): @@ -125,10 +125,12 @@ def _provider_status(e: OpenAIError | str) -> int: return 502 -def model_error(e: OpenAIError | str, *, code: int | None = None) -> ProviderError: +def model_error( + e: OpenAIError | str, *, status_code: int | None = None +) -> ProviderError: """Map a provider failure to our error type: an overlong prompt (a budget limit the interception server turns into a clean truncation) is told apart from any other provider call failure, which - becomes a plain `ProviderError`. `code` is the HTTP status surfaced to the harness (whose + becomes a plain `ProviderError`. `status_code` is the HTTP status surfaced to the harness (whose SDK then retries 5xx/429/timeout and not 4xx); derived from an SDK error when not given. Accepts an SDK error (the renderer) or the provider's raw error body (the httpx proxy).""" from openai import APIStatusError @@ -138,7 +140,12 @@ def model_error(e: OpenAIError | str, *, code: int | None = None) -> ProviderErr if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES): # Keep the provider's real status when the failure carried one; else the class # default (the 400 the interception server surfaces for overlong prompts). - if code is None and isinstance(e, APIStatusError): - code = e.status_code - return OverlongPromptError(text, **({} if code is None else {"code": code})) - return ProviderError(text, code=code if code is not None else _provider_status(e)) + if status_code is None and isinstance(e, APIStatusError): + status_code = e.status_code + return OverlongPromptError( + text, **({} if status_code is None else {"status_code": status_code}) + ) + return ProviderError( + text, + status_code=status_code if status_code is not None else _provider_status(e), + ) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index cbfb53ee53..ceeb7f888c 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -230,7 +230,7 @@ def _fail( ) return web.json_response( dialect.error_body(str(error)), - status=getattr(error, "code", 502), + status=getattr(error, "status_code", 502), ) def record_call( @@ -267,7 +267,9 @@ def record_call( finish_reason=finish_reason, # Every failure surfaces an HTTP status to the harness — the error's own # when it carries one, else the generic 502 the handlers return. - status=getattr(error, "code", 502) if error is not None else None, + status=getattr(error, "status_code", 502) + if error is not None + else None, time=TimeSpan(start=started, end=time.time()), error=None if error is None @@ -463,7 +465,7 @@ def serve(response: Response) -> web.Response: ) return web.json_response( dialect.error_body(str(e)), - status=getattr(e, "code", 502), + status=getattr(e, "status_code", 502), ) except Exception as e: # surface to the program as an API error error = e @@ -589,7 +591,7 @@ async def _stream( e, ) return web.json_response( - dialect.error_body(str(e)), status=getattr(e, "code", 502) + dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) except Exception as e: # surface to the program as an API error error = e @@ -715,7 +717,7 @@ async def handle_aux( e, ) return web.json_response( - dialect.error_body(str(e)), status=getattr(e, "code", 502) + dialect.error_body(str(e)), status=getattr(e, "status_code", 502) ) except Exception as e: logger.warning("aux call failed: id=%s %s", session.trace.id, e) From 50f007b7f0e2b0fb6e68edac2f00a521f86605de Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:34:38 +0000 Subject: [PATCH 23/34] feat(v1)!: finish_reason lives on the call, not the node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No consumer reads MessageNode.finish_reason (prime-rl and research-environments have none; is_truncated was the only reader) and it is an exchange attribute, so it moves to ModelCall exclusively. is_truncated reads the last successful call; the v0 bridge records a minimal per-step call to keep it working. TRACE_VERSION bumps to 2 — nodes of previously persisted traces no longer validate. Co-Authored-By: Claude Fable 5 --- verifiers/v1/graph.py | 4 ---- verifiers/v1/legacy.py | 10 +++++++--- verifiers/v1/trace.py | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index 8f2222acad..d9c25934af 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -31,7 +31,6 @@ from verifiers.v1.types import ( AssistantMessage, - FinishReason, KeptTokens, Message, Response, @@ -104,8 +103,6 @@ class MessageNode(StrictBaseModel): logprobs: list[float] = Field(default_factory=list) """Sampling logprobs for the sampled tokens — length equals the number of True entries in `mask`; empty for input messages.""" - finish_reason: FinishReason = None - """The response's finish reason (assistant nodes only) — kept for truncation detection.""" multi_modal_data: SkipJsonSchema[MultiModalData | None] = None """The renderer items for the images this message's content introduces (pixel tensors, grids, hashes, placeholders) — the only carrier of the pixels from the env server to the @@ -575,7 +572,6 @@ def _commit_turn(turn: PendingTurn, response: Response) -> int: else [], # TurnTokens is discarded after commit, so transfer its logprobs without copying. logprobs=tokens.completion_logprobs if tokens else [], - finish_reason=response.finish_reason, usage=response.usage, ) ) diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index eafa6a0a0f..5ddc562afc 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -32,7 +32,7 @@ ) from verifiers.v1.task import WireTaskData from verifiers.v1 import graph -from verifiers.v1.trace import Error, TimeSpan, Timing, Trace, TraceTask +from verifiers.v1.trace import Error, ModelCall, TimeSpan, Timing, Trace, TraceTask from verifiers.v1.types import ( AssistantMessage, Response, @@ -266,9 +266,13 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace: if not isinstance(step, dict): continue tokens = _to_v1_tokens(step.get("tokens")) - graph.prepare_turn(trace, _to_v1_messages(step.get("prompt"))).commit( - _to_v1_response(step.get("response"), model, tokens) + response = _to_v1_response(step.get("response"), model, tokens) + node = graph.prepare_turn(trace, _to_v1_messages(step.get("prompt"))).commit( + response ) + # The per-call record (v0 steps carry no wire settings or timing): keeps + # `finish_reason` — per-call since trace v2 — available to `is_truncated`. + trace.calls.append(ModelCall(node=node, finish_reason=response.finish_reason)) return trace diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 83bff938ff..15a8765bc5 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -247,9 +247,10 @@ def num_input_tokens(self) -> int: """Raw tensor fields kept on the msgpack wire but excluded from JSON records.""" -TRACE_VERSION = 1 +TRACE_VERSION = 2 """Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on -breaking shape changes; optional-with-default fields are additive and don't bump it.""" +breaking shape changes (v2: `finish_reason` moved from `MessageNode` to `ModelCall`); +optional-with-default fields are additive and don't bump it.""" class EvalRunInfo(StrictBaseModel): @@ -442,7 +443,7 @@ def is_truncated(self) -> bool: "harness_timeout", ): return True - last = self._last_assistant() + last = next((c for c in reversed(self.calls) if c.error is None), None) return bool(last and last.finish_reason == "length") @property From ac77409e46324e17c3a4115400b50b09fc6224b1 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:35:03 +0000 Subject: [PATCH 24/34] docs: ModelCall.status is the upstream status, not the relayed one Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 15a8765bc5..e7624bc4db 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -85,9 +85,11 @@ class ModelCall(StrictBaseModel): """Why the model stopped, normalized (`stop` / `length` / `tool_calls`); None for a failed call or an unrecognized provider reason.""" status: int | None = None - """The HTTP status a failed exchange surfaced (the provider's, one chosen for a - transport fault, or the generic 502 for an unexpected failure); None on success — - a recorded turn implies a 2xx exchange.""" + """The upstream HTTP status of a failed exchange (the provider's own, one chosen + for a transport fault, or the generic 502 for an unexpected failure). Provider-side + truth: it may differ from what the framework relays to the harness (an overlong + prompt is relayed as a 400 whatever the provider said). None on success — a + recorded turn implies a 2xx exchange.""" time: TimeSpan = Field(default_factory=TimeSpan) """Wall-clock span from sending the request to the fully received response.""" error: Error | None = None From bae1e04f57eccb5e55b5c97f4a260bcc65260f84 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:41:59 +0000 Subject: [PATCH 25/34] feat(v1)!: usage lives on the call, not the node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provider usage is an exchange attribute: it moves to ModelCall exclusively (the v0 bridge records it on its per-step call). Branches keep their token accounting — Trace.branches attaches each branch's calls in path order, and Branch.usage/last_usage read those — so trace/branch num_* properties and the platform push payload are unchanged. Joins the v2 trace schema. Co-Authored-By: Claude Fable 5 --- verifiers/v1/graph.py | 5 ----- verifiers/v1/interception/server.py | 6 +++++- verifiers/v1/legacy.py | 11 +++++++++-- verifiers/v1/trace.py | 26 ++++++++++++++++++++------ 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index d9c25934af..506deda90e 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -38,7 +38,6 @@ TextContentPart, Tool, ToolMessage, - Usage, ) if TYPE_CHECKING: @@ -109,9 +108,6 @@ class MessageNode(StrictBaseModel): trainer. `Branch.multi_modal_data` concatenates them along the path into the training `mm_kwargs`. Rides the wire as raw bytes (msgpack `bin`) since pydantic can't JSON the numpy; kept off disk by the dump-site `exclude` in prime-rl (the tensors bloat the rollout jsonl).""" - usage: Usage | None = None - """Provider-reported token usage for this message's response (assistant nodes). Preserved - on the wire and on disk, including cache-read tokens when the provider reports them.""" routed_experts: SkipJsonSchema[np.ndarray | None] = None """This node's slice of the MoE expert-routing array — uint8 `[len(token_ids), layers, top_k]`, the expert ids inference selected for exactly this node's tokens. Attributed from @@ -572,7 +568,6 @@ def _commit_turn(turn: PendingTurn, response: Response) -> int: else [], # TurnTokens is discarded after commit, so transfer its logprobs without copying. logprobs=tokens.completion_logprobs if tokens else [], - usage=response.usage, ) ) # Register the assistant so the next turn's prompt (which restates it) reuses this node. diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index ceeb7f888c..910f079de9 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -54,7 +54,7 @@ ) from verifiers.v1.session import RolloutSession from verifiers.v1.trace import Error, ModelCall, TimeSpan -from verifiers.v1.types import FinishReason, Messages, Response, Tool +from verifiers.v1.types import FinishReason, Messages, Response, Tool, Usage logger = logging.getLogger(__name__) @@ -242,6 +242,7 @@ def record_call( *, node: int | None = None, finish_reason: "FinishReason" = None, + usage: "Usage | None" = None, error: BaseException | None = None, ) -> None: """Append one provider exchange to the trace's per-call records (`Trace.calls`): @@ -265,6 +266,7 @@ def record_call( sampling=sampling, endpoint=dialect.upstream_path, finish_reason=finish_reason, + usage=usage, # Every failure surfaces an HTTP status to the harness — the error's own # when it carries one, else the generic 502 the handlers return. status=getattr(error, "status_code", 502) @@ -493,6 +495,7 @@ def serve(response: Response) -> web.Response: finish_reason=call_response.finish_reason if call_response else None, + usage=call_response.usage if call_response else None, error=error, ) # Hand back to the program when the model wants a tool (the program runs it) or @@ -691,6 +694,7 @@ async def _stream( started, node=node, finish_reason=response.finish_reason if response is not None else None, + usage=response.usage if response is not None else None, error=error, ) diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index 5ddc562afc..cb7aa6d731 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -271,8 +271,15 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace: response ) # The per-call record (v0 steps carry no wire settings or timing): keeps - # `finish_reason` — per-call since trace v2 — available to `is_truncated`. - trace.calls.append(ModelCall(node=node, finish_reason=response.finish_reason)) + # `finish_reason` and `usage` — per-call since trace v2 — available to + # `is_truncated` and the token accounting. + trace.calls.append( + ModelCall( + node=node, + finish_reason=response.finish_reason, + usage=response.usage, + ) + ) return trace diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index e7624bc4db..138f9b4d47 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -90,6 +90,9 @@ class ModelCall(StrictBaseModel): truth: it may differ from what the framework relays to the harness (an overlong prompt is relayed as a 400 whatever the provider said). None on success — a recorded turn implies a 2xx exchange.""" + usage: Usage | None = None + """Provider-reported token usage for this exchange, cache reads included; None for + a failed call.""" time: TimeSpan = Field(default_factory=TimeSpan) """Wall-clock span from sending the request to the fully received response.""" error: Error | None = None @@ -102,6 +105,9 @@ class Branch(StrictBaseModel): index: int nodes: list[MessageNode] + calls: list[ModelCall] = Field(default_factory=list) + """The exchanges behind this branch's sampled turns, in path order — attached by + `Trace.branches` (a derived view, like the branch itself).""" @property def num_turns(self) -> int: @@ -206,13 +212,13 @@ def kept_tokens(self) -> KeptTokens | None: @property def usage(self) -> Usage | None: - return Usage.aggregate(n.usage for n in self.nodes if n.usage is not None) + return Usage.aggregate(c.usage for c in self.calls if c.usage is not None) @property def last_usage(self) -> Usage | None: """Provider usage from the final model call on this branch — the full context it saw.""" return next( - (n.usage for n in reversed(self.nodes) if n.usage is not None), None + (c.usage for c in reversed(self.calls) if c.usage is not None), None ) @property @@ -251,7 +257,8 @@ def num_input_tokens(self) -> int: TRACE_VERSION = 2 """Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on -breaking shape changes (v2: `finish_reason` moved from `MessageNode` to `ModelCall`); +breaking shape changes (v2: `finish_reason` and `usage` moved from `MessageNode` to +`ModelCall`); optional-with-default fields are additive and don't bump it.""" @@ -402,7 +409,7 @@ def num_total_tokens(self) -> int: @property def usage(self) -> Usage | None: """Provider-reported usage summed once per actual model call in this rollout.""" - return Usage.aggregate(n.usage for n in self.nodes if n.usage is not None) + return Usage.aggregate(c.usage for c in self.calls if c.usage is not None) @property def has_response(self) -> bool: @@ -411,7 +418,8 @@ def has_response(self) -> bool: @property def branches(self) -> list[Branch]: - """One root-to-leaf path per graph leaf.""" + """One root-to-leaf path per graph leaf, its calls attached in path order.""" + by_node = {c.node: c for c in self.calls if c.node is not None} branches: list[Branch] = [] for i, leaf in enumerate(graph.leaves(self)): path: list[int] = [] @@ -420,7 +428,13 @@ def branches(self) -> list[Branch]: path.append(nid) nid = self.nodes[nid].parent path.reverse() - branches.append(Branch(index=i, nodes=[self.nodes[n] for n in path])) + branches.append( + Branch( + index=i, + nodes=[self.nodes[n] for n in path], + calls=[by_node[n] for n in path if n in by_node], + ) + ) return branches @property From 4d3edb8294585840ffa5dd6af03235b0a8740794 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:47:08 +0000 Subject: [PATCH 26/34] fix(v1): canonicalize the chat max-tokens alias on call records Both max_tokens and max_completion_tokens can ride one wire request (an eval override on top of a harness's alias); the record keeps the canonical knob only, override winning. Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/chat.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/verifiers/v1/dialects/chat.py b/verifiers/v1/dialects/chat.py index fce693ca23..5aac072752 100644 --- a/verifiers/v1/dialects/chat.py +++ b/verifiers/v1/dialects/chat.py @@ -20,6 +20,7 @@ Message, Messages, Response, + Sampling, SamplingConfig, SystemMessage, Tool, @@ -331,6 +332,14 @@ def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]: tool_names[call.id] = call.name return messages, parse_tools(body.get("tools")) + def parse_sampling(self, body: dict) -> Sampling: + settings = {k: v for k, v in body.items() if k in self.sampling_fields} + # Canonicalize the max-tokens alias; when both ride the wire (an eval override + # on top of a harness's `max_completion_tokens`), the override wins. + if (mct := settings.pop("max_completion_tokens", None)) is not None: + settings.setdefault("max_tokens", mct) + return Sampling.model_validate(settings) + def parse_response(self, response: ChatCompletion) -> Response: return response_from_wire(response) From fd3c22e78ca5b3095a13b06fcd5733117d5977f9 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 17 Jul 2026 18:51:16 -0700 Subject: [PATCH 27/34] move calls --- tests/v1/test_dialects.py | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 tests/v1/test_dialects.py diff --git a/tests/v1/test_dialects.py b/tests/v1/test_dialects.py new file mode 100644 index 0000000000..742dab626c --- /dev/null +++ b/tests/v1/test_dialects.py @@ -0,0 +1,62 @@ +"""Dialect response validation: preserving provider-specific `service_tier` values. + +OpenAI pins `service_tier` to a closed Literal, but OpenAI-compatible gateways (e.g. OpenRouter) +report their own tiers like `provisioned` or `openai/flex`. vf must stay agnostic: preserve the +provider's value rather than rejecting the response or dropping the field. +""" + +import pytest + +from verifiers.v1.dialects.anthropic import AnthropicDialect +from verifiers.v1.dialects.chat import ChatDialect + + +def _chat_raw(service_tier): + return { + "id": "x", + "object": "chat.completion", + "created": 0, + "model": "google/gemini-3-flash-preview", + "service_tier": service_tier, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + +@pytest.mark.parametrize( + "tier", ["provisioned", "openai/flex", "default", "priority", None] +) +def test_chat_preserves_service_tier(tier): + completion = ChatDialect().validate_response(_chat_raw(tier)) + assert completion.service_tier == tier + # the response still parses end-to-end + assert ChatDialect().parse_response(completion).message.content == "hi" + + +def _anthropic_raw(service_tier): + return { + "id": "x", + "type": "message", + "role": "assistant", + "model": "claude-x", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "service_tier": service_tier, + }, + } + + +@pytest.mark.parametrize("tier", ["provisioned", "standard", "priority", None]) +def test_anthropic_preserves_service_tier(tier): + message = AnthropicDialect().validate_response(_anthropic_raw(tier)) + assert message.usage.service_tier == tier + assert AnthropicDialect().parse_response(message).message.content == "hi" From 7fd980cb4a4bc89daf3495ab4489934f7711e5df Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:53:59 +0000 Subject: [PATCH 28/34] fix(v1): preserve gateway service tiers on Anthropic responses Widen Usage.service_tier to a plain string (the ModdedChatCompletion pattern) instead of stripping unknown tiers before validation, so the new dialect tests' provisioned case round-trips. Co-Authored-By: Claude Fable 5 --- verifiers/v1/dialects/anthropic.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/verifiers/v1/dialects/anthropic.py b/verifiers/v1/dialects/anthropic.py index bf324f1aa2..cfe73a6039 100644 --- a/verifiers/v1/dialects/anthropic.py +++ b/verifiers/v1/dialects/anthropic.py @@ -11,6 +11,7 @@ from dataclasses import dataclass, field from anthropic.types import Message as AnthropicMessage +from anthropic.types import Usage as AnthropicUsage from verifiers.v1.dialects.base import Dialect, StreamParser, parse_sse_event from verifiers.v1.types import ( @@ -243,6 +244,18 @@ def finish(self) -> Response: return response_from_wire(self.validate_response(self.message)) +class ModdedUsage(AnthropicUsage): + """The SDK closes `service_tier` to a fixed Literal, but Anthropic-compatible gateways + report their own tiers (e.g. Prime's `provisioned`). Widen to a plain string — we don't + consume it — so parsing stays lenient about the label instead of dropping it.""" + + service_tier: str | None = None # type: ignore[assignment] + + +class ModdedAnthropicMessage(AnthropicMessage): + usage: ModdedUsage # type: ignore[assignment] + + class AnthropicDialect(Dialect[dict, AnthropicMessage]): sampling_fields = frozenset( { @@ -259,7 +272,7 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]): routes = ("/v1/messages",) aux_routes = ("/v1/messages/count_tokens",) upstream_path = "/v1/messages" - response_type = AnthropicMessage + response_type = ModdedAnthropicMessage def auth_headers(self, api_key: str) -> dict[str, str]: return {"x-api-key": api_key, "anthropic-version": "2023-06-01"} @@ -289,14 +302,6 @@ def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]: def parse_response(self, response: AnthropicMessage) -> Response: return response_from_wire(response) - def validate_response(self, raw: dict) -> AnthropicMessage: - usage = raw.get("usage") - tier = usage.get("service_tier") if usage else None - if tier not in (None, "standard", "priority", "batch"): - raw = {**raw, "usage": usage.copy()} - raw["usage"].pop("service_tier") - return super().validate_response(raw) - def stream_parser(self) -> StreamParser: return AnthropicStreamParser(self.validate_response) From dc2b3f3432b07f9111d7a2b2f1493328cebd5704 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:54:55 +0000 Subject: [PATCH 29/34] chore: drop the dialect unit tests Co-Authored-By: Claude Fable 5 --- tests/v1/test_dialects.py | 62 --------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 tests/v1/test_dialects.py diff --git a/tests/v1/test_dialects.py b/tests/v1/test_dialects.py deleted file mode 100644 index 742dab626c..0000000000 --- a/tests/v1/test_dialects.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Dialect response validation: preserving provider-specific `service_tier` values. - -OpenAI pins `service_tier` to a closed Literal, but OpenAI-compatible gateways (e.g. OpenRouter) -report their own tiers like `provisioned` or `openai/flex`. vf must stay agnostic: preserve the -provider's value rather than rejecting the response or dropping the field. -""" - -import pytest - -from verifiers.v1.dialects.anthropic import AnthropicDialect -from verifiers.v1.dialects.chat import ChatDialect - - -def _chat_raw(service_tier): - return { - "id": "x", - "object": "chat.completion", - "created": 0, - "model": "google/gemini-3-flash-preview", - "service_tier": service_tier, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "hi"}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - - -@pytest.mark.parametrize( - "tier", ["provisioned", "openai/flex", "default", "priority", None] -) -def test_chat_preserves_service_tier(tier): - completion = ChatDialect().validate_response(_chat_raw(tier)) - assert completion.service_tier == tier - # the response still parses end-to-end - assert ChatDialect().parse_response(completion).message.content == "hi" - - -def _anthropic_raw(service_tier): - return { - "id": "x", - "type": "message", - "role": "assistant", - "model": "claude-x", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "usage": { - "input_tokens": 1, - "output_tokens": 1, - "service_tier": service_tier, - }, - } - - -@pytest.mark.parametrize("tier", ["provisioned", "standard", "priority", None]) -def test_anthropic_preserves_service_tier(tier): - message = AnthropicDialect().validate_response(_anthropic_raw(tier)) - assert message.usage.service_tier == tier - assert AnthropicDialect().parse_response(message).message.content == "hi" From 28281009c48f6f5fa45e511ea6c8bf1267db75ca Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:56:32 +0000 Subject: [PATCH 30/34] chore: keep the TRACE_VERSION docstring unversioned Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 138f9b4d47..b5cd85c7e9 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -257,9 +257,7 @@ def num_input_tokens(self) -> int: TRACE_VERSION = 2 """Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on -breaking shape changes (v2: `finish_reason` and `usage` moved from `MessageNode` to -`ModelCall`); -optional-with-default fields are additive and don't bump it.""" +breaking shape changes; optional-with-default fields are additive and don't bump it.""" class EvalRunInfo(StrictBaseModel): From 506274275f8e1da3cd658bc530e80ed9fcb04ebb Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 01:58:48 +0000 Subject: [PATCH 31/34] refactor(v1): status_code lives on Error, not beside it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The status only exists for failures, so it belongs on the failure record: Error gains status_code (populated on per-call records and rollout-level capture_error alike) and ModelCall.status goes away. No 502 fallback — a non-HTTP failure (commit error, cancellation) simply has none. Co-Authored-By: Claude Fable 5 --- verifiers/v1/interception/server.py | 6 +----- verifiers/v1/trace.py | 10 ++++------ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 910f079de9..c76b989a23 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -267,17 +267,13 @@ def record_call( endpoint=dialect.upstream_path, finish_reason=finish_reason, usage=usage, - # Every failure surfaces an HTTP status to the harness — the error's own - # when it carries one, else the generic 502 the handlers return. - status=getattr(error, "status_code", 502) - if error is not None - else None, time=TimeSpan(start=started, end=time.time()), error=None if error is None else Error( type=type(error).__name__, message=str(error), + status_code=getattr(error, "status_code", None), # Provider errors already carry the actionable upstream diagnostic. # Format from the exception object: the record is written in a # `finally`, where the ambient exception state is already cleared. diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index b5cd85c7e9..3bfb18ba3e 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -60,6 +60,9 @@ class Timing(StrictBaseModel): class Error(StrictBaseModel): type: str message: str + status_code: int | None = None + """The upstream HTTP status a provider failure surfaced (the provider's own, or one + chosen for a transport fault); None when the failure carried no HTTP exchange.""" traceback: str | None = None @@ -84,12 +87,6 @@ class ModelCall(StrictBaseModel): finish_reason: FinishReason = None """Why the model stopped, normalized (`stop` / `length` / `tool_calls`); None for a failed call or an unrecognized provider reason.""" - status: int | None = None - """The upstream HTTP status of a failed exchange (the provider's own, one chosen - for a transport fault, or the generic 502 for an unexpected failure). Provider-side - truth: it may differ from what the framework relays to the harness (an overlong - prompt is relayed as a 400 whatever the provider said). None on success — a - recorded turn implies a 2xx exchange.""" usage: Usage | None = None """Provider-reported token usage for this exchange, cache reads included; None for a failed call.""" @@ -546,6 +543,7 @@ def capture_error(self, error: Exception) -> None: Error( type=type(error).__name__, message=str(error), + status_code=getattr(error, "status_code", None), # Provider errors already carry the actionable upstream diagnostic. # Keep full tracebacks for every other failure. traceback=None From cf18f49596b94d08dd8e03da53d71567c5d3815f Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 02:00:41 +0000 Subject: [PATCH 32/34] docs: mention per-call ModelCall records in the trace overview Co-Authored-By: Claude Fable 5 --- docs/v1/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/v1/overview.md b/docs/v1/overview.md index e9c8e96a98..607cb8715d 100644 --- a/docs/v1/overview.md +++ b/docs/v1/overview.md @@ -22,7 +22,7 @@ A set of tools defined by the taskset that are installed as MCP servers into the ## Trace -A trace records the message graph, rewards, metrics, errors, etc. When using verifiers for training with [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl), it stores additional information such as tokens and logprobs, built incrementally using [renderers](https://github.com/PrimeIntellect-ai/renderers). +A trace records the message graph, rewards, metrics, errors, and one per-call record (`ModelCall`) per provider exchange (its model, sampling, finish reason, usage, timing, and any error), etc. When using verifiers for training with [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl), it stores additional information such as tokens and logprobs, built incrementally using [renderers](https://github.com/PrimeIntellect-ai/renderers). ## Documentation From a02c7f5f0c43847b0877a5ae53b5cc4f040b3506 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 02:04:07 +0000 Subject: [PATCH 33/34] docs: add per-call records to the evaluate skill's trace checklist Co-Authored-By: Claude Fable 5 --- skills/evaluate-environments/SKILL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/skills/evaluate-environments/SKILL.md b/skills/evaluate-environments/SKILL.md index 43b0b91bf5..e8ec3ffae4 100644 --- a/skills/evaluate-environments/SKILL.md +++ b/skills/evaluate-environments/SKILL.md @@ -165,6 +165,7 @@ For each representative sample inspect: - named `rewards`, aggregate `reward`, and `metrics`; - persisted `info` artifacts; - `error`/`errors` and boundary type; +- per-call `calls` records (model, sampling, finish reason, usage, timing, error) linked to the graph; - usage and stage timing; - token/mask/logprob fields when using the training client. From ce55eadc508bc5ebc7b35616d81d8cfaaa668817 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 18 Jul 2026 02:05:35 +0000 Subject: [PATCH 34/34] chore: ModelCall.sampling typed as Sampling; drop Branch.calls docstring Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 3bfb18ba3e..bf275e742c 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -26,6 +26,7 @@ FinishReason, KeptTokens, Messages, + Sampling, SamplingConfig, StrictBaseModel, Tool, @@ -77,7 +78,7 @@ class ModelCall(StrictBaseModel): model: str | None = None """The model requested from the provider. The rollout's model override makes this `agent.model` on every call; recorded per call because it is cheap and provable.""" - sampling: SamplingConfig | None = None + sampling: Sampling | None = None """The call's effective settings, scraped off the wire request by the dialect's `sampling_fields` whitelist — the eval-imposed knobs plus whatever the harness set that the eval left alone (`seed`, `tool_choice`, `response_format`, ... as extras).""" @@ -103,8 +104,6 @@ class Branch(StrictBaseModel): index: int nodes: list[MessageNode] calls: list[ModelCall] = Field(default_factory=list) - """The exchanges behind this branch's sampled turns, in path order — attached by - `Trace.branches` (a derived view, like the branch itself).""" @property def num_turns(self) -> int: