feat: record per-call model call metadata on v1 traces

tests/v1/test_e2e.py

@@ -18,6 +18,14 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns == 1
     assert trace.reward == 1.0
+    # Every sampled turn has one raw per-call record, linked to its assistant node.
+    sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
+    assert [c.node for c in trace.calls if c.error is None] == sampled
+    for call in trace.calls:
+        assert call.request is not None
+        assert call.time.duration > 0
+        if call.error is None:
+            assert call.response is not None
 
 
 @pytest.mark.e2e

verifiers/v1/clients/client.py

@@ -26,6 +26,8 @@ class RelayReply:
     content_type: str
     chunks: AsyncIterator[bytes]
     close: Callable[[], Awaitable[None]]
+    headers: dict[str, str] | None = None
+    """Provider response headers, for the trace's per-call records."""
 
 
 class Client(ABC):

verifiers/v1/clients/eval.py

@@ -107,9 +107,11 @@ async def get_response(
             raise model_error(
                 f"malformed upstream response: {type(e).__name__}: {e}",
                 status_code=502,
+                headers=dict(resp.headers),
             ) from e
         # The interception server returns this full native provider object to the program.
         response.raw = raw
+        response.raw_headers = dict(resp.headers)
         return response
 
     def _headers(
@@ -169,6 +171,7 @@ async def _request(
                 raise model_error(
                     f"upstream {e.response.status_code}: {e.response.text}",
                     status_code=e.response.status_code,
+                    headers=dict(e.response.headers),
                 ) from e
             return response
         if response.status_code < 400:
@@ -178,7 +181,9 @@ async def _request(
         finally:
             await response.aclose()
         raise model_error(
-            f"upstream {response.status_code}: {text}", status_code=response.status_code
+            f"upstream {response.status_code}: {text}",
+            status_code=response.status_code,
+            headers=dict(response.headers),
         )
 
     async def relay(
@@ -217,6 +222,7 @@ async def chunks():
             content_type=resp.headers.get("content-type", "text/event-stream"),
             chunks=chunks(),
             close=resp.aclose,
+            headers=dict(resp.headers),
         )
 
     async def relay_aux(

verifiers/v1/dialects/anthropic.py

@@ -239,7 +239,9 @@ def finish(self) -> Response:
         for index, parts in self.partial_json.items():
             self.blocks[index]["input"] = json.loads("".join(parts) or "{}")
         self.message["content"] = [self.blocks[index] for index in sorted(self.blocks)]
-        return response_from_wire(self.validate_response(self.message))
+        response = response_from_wire(self.validate_response(self.message))
+        response.raw = self.message
+        return response
 
 
 class AnthropicDialect(Dialect[dict, AnthropicMessage]):

verifiers/v1/dialects/chat.py

@@ -271,24 +271,23 @@ def finish(self) -> Response:
         if self.reasoning_details:
             self.message["reasoning_details"] = self.reasoning_details
         head = self.head or {}
-        return response_from_wire(
-            ModdedChatCompletion.model_validate(
+        completion = {
+            "id": head.get("id", "vf-intercept"),
+            "object": "chat.completion",
+            "created": head.get("created", int(time.time())),
+            "model": head.get("model", ""),
+            "choices": [
                 {
-                    "id": head.get("id", "vf-intercept"),
-                    "object": "chat.completion",
-                    "created": head.get("created", int(time.time())),
-                    "model": head.get("model", ""),
-                    "choices": [
-                        {
-                            "index": 0,
-                            "message": self.message,
-                            "finish_reason": self.finish_reason or "stop",
-                        }
-                    ],
-                    "usage": self.usage,
+                    "index": 0,
+                    "message": self.message,
+                    "finish_reason": self.finish_reason or "stop",
                 }
-            )
-        )
+            ],
+            "usage": self.usage,
+        }
+        response = response_from_wire(ModdedChatCompletion.model_validate(completion))
+        response.raw = completion
+        return response
 
 
 class ChatDialect(Dialect[dict, ChatCompletion]):

verifiers/v1/dialects/responses.py

@@ -270,9 +270,11 @@ def finish(self) -> Response:
         events = self.terminal_events or self.events
         for event in iter_sse_reverse(b"".join(events)):
             if event.get("type") in FINAL_EVENTS:
-                return response_from_wire(
+                response = response_from_wire(
                     OpenAIResponse.model_validate(event["response"])
                 )
+                response.raw = event["response"]
+                return response
         raise ValueError("Responses stream ended without a terminal event")
 
 

verifiers/v1/errors.py

@@ -37,9 +37,18 @@ class ProviderError(RolloutError):
     (5xx/429/timeout) and not deterministic ones (4xx) — relayed from the provider, or chosen for a
     transport fault."""
 
-    def __init__(self, message: str = "", *, status_code: int = 502) -> None:
+    def __init__(
+        self,
+        message: str = "",
+        *,
+        status_code: int = 502,
+        headers: dict[str, str] | None = None,
+    ) -> None:
         super().__init__(message)
         self.status_code = status_code
+        self.headers = headers
+        """Provider response headers when the failure carried an HTTP response (request
+        ids, rate-limit diagnostics) — surfaced on the trace's per-call records."""
 
 
 class OverlongPromptError(ProviderError):
@@ -121,7 +130,10 @@ def _provider_status(e: OpenAIError | str) -> int:
 
 
 def model_error(
-    e: OpenAIError | str, *, status_code: int | None = None
+    e: OpenAIError | str,
+    *,
+    status_code: int | None = None,
+    headers: dict[str, str] | None = None,
 ) -> ProviderError:
     """Map a provider failure to our error type: an overlong prompt (a budget limit the interception
     server turns into a clean truncation) is told apart from any other provider call failure, which
@@ -131,8 +143,9 @@ def model_error(
     # Some SDK errors stringify empty; fall back to the type so the message is never blank.
     text = str(e) or (type(e).__name__ if isinstance(e, BaseException) else "")
     if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES):
-        return OverlongPromptError(text)
+        return OverlongPromptError(text, headers=headers)
     return ProviderError(
         text,
         status_code=status_code if status_code is not None else _provider_status(e),
+        headers=headers,
     )

verifiers/v1/graph.py

@@ -343,10 +343,12 @@ def prompt_message_spans(
             for span in tail_spans
         ]
 
-    def commit(self, response: Response, tools: list[Tool] | None = None) -> None:
-        _commit_turn(self, response)
+    def commit(self, response: Response, tools: list[Tool] | None = None) -> int:
+        """Add this turn to the graph; returns the committed assistant node's id."""
+        assistant_id = _commit_turn(self, response)
         if tools:
             self.trace.tools = tools
+        return assistant_id
 
 
 def prepare_turn(trace: Trace, prompt: list[Message]) -> PendingTurn:
@@ -486,7 +488,7 @@ def _attribute_kept_tokens(
     node.kept_tokens = KeptTokens(ids=ids.copy(), counts=counts.copy())
 
 
-def _commit_turn(turn: PendingTurn, response: Response) -> None:
+def _commit_turn(turn: PendingTurn, response: Response) -> int:
     trace = turn.trace
     prompt = turn.prompt
     tokens = response.tokens
@@ -596,6 +598,8 @@ def _commit_turn(turn: PendingTurn, response: Response) -> None:
     # completion-aligned, so only the sampled node carries them).
     _attribute_kept_tokens(trace, assistant_id, tokens.kept_tokens if tokens else None)
 
+    return assistant_id
+
 
 # --- walking the graph (views) ---------------------------------------------------------