feat: record per-call model call metadata on v1 traces

docs/v1/overview.md

@@ -22,7 +22,7 @@ A set of tools defined by the taskset that are installed as MCP servers into the
 
 ## Trace
 
-A trace records the message graph, rewards, metrics, errors, etc. When using verifiers for training with [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl), it stores additional information such as tokens and logprobs, built incrementally using [renderers](https://github.com/PrimeIntellect-ai/renderers).
+A trace records the message graph, rewards, metrics, errors, and one per-call record (`ModelCall`) per provider exchange (its model, sampling, finish reason, usage, timing, and any error), etc. When using verifiers for training with [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl), it stores additional information such as tokens and logprobs, built incrementally using [renderers](https://github.com/PrimeIntellect-ai/renderers).
 
 ## Documentation
 

skills/evaluate-environments/SKILL.md

@@ -165,6 +165,7 @@ For each representative sample inspect:
 - named `rewards`, aggregate `reward`, and `metrics`;
 - persisted `info` artifacts;
 - `error`/`errors` and boundary type;
+- per-call `calls` records (model, sampling, finish reason, usage, timing, error) linked to the graph;
 - usage and stage timing;
 - token/mask/logprob fields when using the training client.
 

tests/v1/test_e2e.py

@@ -18,6 +18,12 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns == 1
     assert trace.reward == 1.0
+    # Every sampled turn has one per-call record, linked to its assistant node.
+    sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
+    assert [c.node for c in trace.calls if c.error is None] == sampled
+    for call in trace.calls:
+        assert call.model and call.sampling is not None
+        assert call.time.duration > 0
 
 
 @pytest.mark.e2e

verifiers/v1/__init__.py

@@ -105,6 +105,7 @@
     Branch,
     Error,
     EvalRunInfo,
+    ModelCall,
     RunInfo,
     TimeSpan,
     Timing,
@@ -172,6 +173,7 @@
     "AgentInfo",
     "RunInfo",
     "EvalRunInfo",
+    "ModelCall",
     "TrainRunInfo",
     "VersionInfo",
     "State",

verifiers/v1/dialects/anthropic.py

@@ -11,6 +11,7 @@
 from dataclasses import dataclass, field
 
 from anthropic.types import Message as AnthropicMessage
+from anthropic.types import Usage as AnthropicUsage
 
 from verifiers.v1.dialects.base import Dialect, StreamParser, parse_sse_event
 from verifiers.v1.types import (
@@ -21,6 +22,7 @@
     ImageUrlSource,
     Messages,
     Response,
+    Sampling,
     SamplingConfig,
     SystemMessage,
     TextContentPart,
@@ -242,11 +244,35 @@ def finish(self) -> Response:
         return response_from_wire(self.validate_response(self.message))
 
 
+class ModdedUsage(AnthropicUsage):
+    """The SDK closes `service_tier` to a fixed Literal, but Anthropic-compatible gateways
+    report their own tiers (e.g. Prime's `provisioned`). Widen to a plain string — we don't
+    consume it — so parsing stays lenient about the label instead of dropping it."""
+
+    service_tier: str | None = None  # type: ignore[assignment]
+
+
+class ModdedAnthropicMessage(AnthropicMessage):
+    usage: ModdedUsage  # type: ignore[assignment]
+
+
 class AnthropicDialect(Dialect[dict, AnthropicMessage]):
+    sampling_fields = frozenset(
+        {
+            "temperature",
+            "top_p",
+            "top_k",
+            "max_tokens",
+            "stop_sequences",
+            "thinking",
+            "tool_choice",
+            "output_config",
+        }
+    )
     routes = ("/v1/messages",)
     aux_routes = ("/v1/messages/count_tokens",)
     upstream_path = "/v1/messages"
-    response_type = AnthropicMessage
+    response_type = ModdedAnthropicMessage
 
     def auth_headers(self, api_key: str) -> dict[str, str]:
         return {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
@@ -276,17 +302,23 @@ def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]:
     def parse_response(self, response: AnthropicMessage) -> Response:
         return response_from_wire(response)
 
-    def validate_response(self, raw: dict) -> AnthropicMessage:
-        usage = raw.get("usage")
-        tier = usage.get("service_tier") if usage else None
-        if tier not in (None, "standard", "priority", "batch"):
-            raw = {**raw, "usage": usage.copy()}
-            raw["usage"].pop("service_tier")
-        return super().validate_response(raw)
-
     def stream_parser(self) -> StreamParser:
         return AnthropicStreamParser(self.validate_response)
 
+    def parse_sampling(self, body: dict) -> Sampling:
+        settings = {k: v for k, v in body.items() if k in self.sampling_fields}
+        # Lift `output_config.effort` (where `apply_overrides` puts the eval's
+        # reasoning effort) onto the typed knob; keep any other output-config keys.
+        if isinstance(config := settings.get("output_config"), dict):
+            config = dict(config)
+            if config.get("effort"):
+                settings["reasoning_effort"] = config.pop("effort")
+            if config:
+                settings["output_config"] = config
+            else:
+                settings.pop("output_config")
+        return Sampling.model_validate(settings)
+
     def apply_overrides(self, body: dict, model: str, sampling: SamplingConfig) -> dict:
         # Preserve native fields except the eval's model + sampling. `temperature`/`top_p` are
         # authoritative (always dropped, the eval's applied if set); `max_tokens` is required by

verifiers/v1/dialects/base.py

@@ -21,7 +21,7 @@
 from pydantic import BaseModel
 from pydantic_core import from_json
 
-from verifiers.v1.types import Messages, Response, SamplingConfig, Tool
+from verifiers.v1.types import Messages, Response, Sampling, SamplingConfig, Tool
 
 ReqT = TypeVar("ReqT")
 RespT = TypeVar("RespT", bound=BaseModel)
@@ -121,6 +121,12 @@ class Dialect(ABC, Generic[ReqT, RespT]):
     `dialects.DIALECTS` and a harness speaking that format works end-to-end (the eval client and
     interception server are generic over this interface)."""
 
+    sampling_fields: ClassVar[frozenset[str]] = frozenset()
+    """Request keys that are call settings — what shapes generation given the same
+    conversation: decoding knobs, budgets/stops, reasoning effort, output contract.
+    A whitelist, so payload, conversation state, and tracking fields can never leak
+    into the per-call record by omission; an unlisted knob is simply not recorded."""
+
     routes: ClassVar[tuple[str, ...]]
     """The endpoint path(s) a program's SDK posts model turns to. The interception server serves
     one handler per route, so the wire format is resolved from the route the SDK chose (it
@@ -167,6 +173,14 @@ def error_body(self, message: str) -> dict:
     def parse_request(self, body: ReqT) -> tuple[Messages, list[Tool] | None]:
         """The native request -> vf prompt + tools (for the trace)."""
 
+    def parse_sampling(self, body: ReqT) -> Sampling:
+        """The native request's call settings -> the canonical `Sampling` (for the
+        trace's per-call records): the `sampling_fields` whitelist, with this format's
+        aliases mapped onto the typed knobs; dialect-specific keys ride as extras."""
+        return Sampling.model_validate(
+            {k: v for k, v in body.items() if k in self.sampling_fields}
+        )
+
     @abstractmethod
     def parse_response(self, response: RespT) -> Response:
         """A native (non-streamed) response -> the vf `Response` we consume."""

verifiers/v1/dialects/chat.py

@@ -20,6 +20,7 @@
     Message,
     Messages,
     Response,
+    Sampling,
     SamplingConfig,
     SystemMessage,
     Tool,
@@ -271,27 +272,47 @@ def finish(self) -> Response:
         if self.reasoning_details:
             self.message["reasoning_details"] = self.reasoning_details
         head = self.head or {}
-        return response_from_wire(
-            ModdedChatCompletion.model_validate(
+        completion = {
+            "id": head.get("id", "vf-intercept"),
+            "object": "chat.completion",
+            "created": head.get("created", int(time.time())),
+            "model": head.get("model", ""),
+            "choices": [
                 {
-                    "id": head.get("id", "vf-intercept"),
-                    "object": "chat.completion",
-                    "created": head.get("created", int(time.time())),
-                    "model": head.get("model", ""),
-                    "choices": [
-                        {
-                            "index": 0,
-                            "message": self.message,
-                            "finish_reason": self.finish_reason or "stop",
-                        }
-                    ],
-                    "usage": self.usage,
+                    "index": 0,
+                    "message": self.message,
+                    "finish_reason": self.finish_reason or "stop",
                 }
-            )
-        )
+            ],
+            "usage": self.usage,
+        }
+        return response_from_wire(ModdedChatCompletion.model_validate(completion))
 
 
 class ChatDialect(Dialect[dict, ChatCompletion]):
+    sampling_fields = frozenset(
+        {
+            "temperature",
+            "top_p",
+            "top_k",
+            "min_p",
+            "max_tokens",
+            "max_completion_tokens",
+            "reasoning_effort",
+            "seed",
+            "stop",
+            "n",
+            "logprobs",
+            "top_logprobs",
+            "logit_bias",
+            "frequency_penalty",
+            "presence_penalty",
+            "repetition_penalty",
+            "response_format",
+            "tool_choice",
+            "parallel_tool_calls",
+        }
+    )
     routes = ("/v1/chat/completions",)
     upstream_path = "/chat/completions"
     response_type = ModdedChatCompletion
@@ -311,6 +332,14 @@ def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]:
                     tool_names[call.id] = call.name
         return messages, parse_tools(body.get("tools"))
 
+    def parse_sampling(self, body: dict) -> Sampling:
+        settings = {k: v for k, v in body.items() if k in self.sampling_fields}
+        # Canonicalize the max-tokens alias; when both ride the wire (an eval override
+        # on top of a harness's `max_completion_tokens`), the override wins.
+        if (mct := settings.pop("max_completion_tokens", None)) is not None:
+            settings.setdefault("max_tokens", mct)
+        return Sampling.model_validate(settings)
+
     def parse_response(self, response: ChatCompletion) -> Response:
         return response_from_wire(response)
 

verifiers/v1/dialects/responses.py

@@ -32,6 +32,7 @@
     ImageUrlSource,
     Messages,
     Response,
+    Sampling,
     SamplingConfig,
     SystemMessage,
     TextContentPart,
@@ -277,6 +278,20 @@ def finish(self) -> Response:
 
 
 class ResponsesDialect(Dialect[dict, OpenAIResponse]):
+    sampling_fields = frozenset(
+        {
+            "temperature",
+            "top_p",
+            "max_output_tokens",
+            "max_tool_calls",
+            "reasoning",
+            "text",
+            "tool_choice",
+            "parallel_tool_calls",
+            "top_logprobs",
+            "truncation",
+        }
+    )
     routes = ("/v1/responses",)
     upstream_path = "/responses"
     response_type = OpenAIResponse
@@ -286,6 +301,22 @@ def is_terminal_event(self, chunk: bytes) -> bool:
         # trailing `[DONE]`, so the turn-ending event is the final event, not the sentinel.
         return any(marker in chunk for marker in _TERMINAL_MARKERS)
 
+    def parse_sampling(self, body: dict) -> Sampling:
+        settings = {k: v for k, v in body.items() if k in self.sampling_fields}
+        # Lift `reasoning.effort` onto the typed knob; keep any other reasoning keys
+        # (e.g. `summary`) as the wire sent them.
+        if isinstance(reasoning := settings.get("reasoning"), dict):
+            reasoning = dict(reasoning)
+            if reasoning.get("effort"):
+                settings["reasoning_effort"] = reasoning.pop("effort")
+            if reasoning:
+                settings["reasoning"] = reasoning
+            else:
+                settings.pop("reasoning")
+        if "max_output_tokens" in settings:
+            settings["max_tokens"] = settings.pop("max_output_tokens")
+        return Sampling.model_validate(settings)
+
     def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]:
         prompt: Messages = []
         if instructions := body.get("instructions"):

verifiers/v1/errors.py

@@ -44,7 +44,12 @@ def __init__(self, message: str = "", *, status_code: int = 502) -> None:
 
 class OverlongPromptError(ProviderError):
     """The prompt exceeded the model's context window — a budget limit, ended as a clean
-    truncation rather than recorded as an error."""
+    truncation rather than recorded as an error. Defaults to a 400 (what the interception
+    server surfaces for it — deterministic, so an SDK never retries it); `model_error`
+    keeps the provider's real status when the failure carried one."""
+
+    def __init__(self, message: str = "", *, status_code: int = 400) -> None:
+        super().__init__(message, status_code=status_code)
 
 
 class HarnessError(RolloutError):
@@ -128,10 +133,18 @@ def model_error(
     becomes a plain `ProviderError`. `status_code` is the HTTP status surfaced to the harness (whose
     SDK then retries 5xx/429/timeout and not 4xx); derived from an SDK error when not given. Accepts
     an SDK error (the renderer) or the provider's raw error body (the httpx proxy)."""
+    from openai import APIStatusError
+
     # Some SDK errors stringify empty; fall back to the type so the message is never blank.
     text = str(e) or (type(e).__name__ if isinstance(e, BaseException) else "")
     if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES):
-        return OverlongPromptError(text)
+        # Keep the provider's real status when the failure carried one; else the class
+        # default (the 400 the interception server surfaces for overlong prompts).
+        if status_code is None and isinstance(e, APIStatusError):
+            status_code = e.status_code
+        return OverlongPromptError(
+            text, **({} if status_code is None else {"status_code": status_code})
+        )
     return ProviderError(
         text,
         status_code=status_code if status_code is not None else _provider_status(e),