Unify evaluation and training inference clients

tests/integration/conftest.py

@@ -16,7 +16,7 @@
 import pytest
 from grid import agent_seats, runtime_config
 
-from verifiers.v1.clients import EvalClientConfig, ModelContext
+from verifiers.v1.clients import ClientConfig, ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.episode import Episode
@@ -201,7 +201,7 @@ async def live_ctx():
     # Endpoint config only — each rollout builds and closes its own client.
     yield ModelContext(
         model=CI_MODEL,
-        client=EvalClientConfig(),
+        client=ClientConfig(),
         sampling=SamplingConfig(max_tokens=2048, reasoning_effort=CI_REASONING_EFFORT),
     )
 
@@ -283,7 +283,7 @@ async def answer(request: web.Request) -> web.StreamResponse:
             if request.path.endswith(suffix)
         )
         raw = build(str(body.get("model", "scripted")))
-        if not dialect.streaming(body):
+        if not body.get("stream"):
             return web.json_response(raw)
         response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
         await response.prepare(request)

tests/unit/test_graph.py

@@ -539,20 +539,13 @@ def second_turn(trace, prompt_ids):
     ]
 
 
-def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
+def test_expanded_prompt_tokens_and_attribution_are_reused():
     trace = vf.Trace(
         agent=vf.AgentInfo(config=vf.AgentConfig()),
         task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
     )
     user = vf.UserMessage(content="image")
     assistant = vf.AssistantMessage(content="looked")
-    assert graph.MessageNode(message=user, token_ids=[9]).logical_ids == [9]
-    assert (
-        graph.MessageNode(
-            message=user, token_ids=[9], renderer_token_ids=[]
-        ).logical_ids
-        == []
-    )
     graph.prepare_turn(trace, [user]).commit(
         vf.Response(
             id="a",
@@ -562,11 +555,9 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
             finish_reason="stop",
             tokens=TurnTokens(
                 prompt_ids=[1, 9, 9, 3],
-                renderer_prompt_ids=[1, 9, 3],
                 completion_ids=[4],
-                message_spans=[(0, 2)],
-                is_content=[False, True, False],
-                mm_placeholders=[(1, 2)],
+                message_spans=[(0, 3)],
+                is_content=[False, True, True, False],
                 mm_token_type_id_map={9: 1},
             ),
         )
@@ -577,25 +568,10 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
     assert trace.nodes[0].token_ids == [1, 9, 9]
     assert trace.nodes[0].is_content == [False, True, True]
     turn = graph.prepare_turn(trace, [user, assistant, vf.UserMessage(content="next")])
-    assert turn.previous_renderer_token_ids() == ([1, 9, 3], [4])
-
-    with pytest.raises(ValueError, match="exactly extend"):
-        turn.commit(
-            vf.Response(
-                id="b",
-                created=0,
-                model="t",
-                message=vf.AssistantMessage(content="bad"),
-                finish_reason="stop",
-                tokens=TurnTokens(
-                    prompt_ids=[1, 9, 8, 3, 4, 5],
-                    renderer_prompt_ids=[1, 9, 3, 4, 5],
-                    bridged=True,
-                    completion_ids=[6],
-                    message_spans=[None, None, (4, 5)],
-                ),
-            )
-        )
+    assert [trace.nodes[nid].token_ids for nid in turn.prefix_node_ids] == [
+        [1, 9, 9],
+        [3, 4],
+    ]
 
     tool = vf.ToolMessage(content="result", tool_call_id="call_0")
     graph.prepare_turn(trace, [user, assistant, tool]).commit(
@@ -607,12 +583,9 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
             finish_reason="stop",
             tokens=TurnTokens(
                 prompt_ids=[1, 9, 9, 3, 4, 5, 6, 7],
-                renderer_prompt_ids=[1, 9, 3, 4, 5, 6, 7],
-                bridged=True,
                 completion_ids=[8],
-                message_spans=[None, None, (4, 6)],
-                is_content=[False, True, False, True, False, True, False],
-                mm_placeholders=[(1, 2)],
+                message_spans=[None, None, (5, 7)],
+                is_content=[False, True, True, False, True, False, True, False],
                 mm_token_type_id_map={9: 1},
             ),
         )
@@ -621,7 +594,6 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
     tool_node = trace.nodes[-2]
     assert tool_node.message == tool
     assert tool_node.token_ids == [5, 6]
-    assert tool_node.renderer_token_ids == [5, 6]
     assert tool_node.is_content == [False, True]
 
 

tests/unit/test_trace.py

@@ -243,12 +243,12 @@ def test_transcript(make_full_trace):
     assert "SECRET REASONING" not in transcript  # reasoning is excluded
 
 
-def test_sampling_mask_from_packed_matches_lists():
+def test_packed_training_records_match_native_arrays():
     import base64
 
     import numpy as np
 
-    from verifiers.v1.types import SamplingMask
+    from verifiers.v1.dialects.chat import response_from_wire
 
     rows = [[4, 7], [], [9]]
     ids, counts = np.array([4, 7, 9], np.int32), np.array([2, 0, 1], np.int32)
@@ -260,9 +260,46 @@ def test_sampling_mask_from_packed_matches_lists():
         }
         for name, a in (("ids", ids), ("counts", counts))
     }
-    for mask in (
-        SamplingMask.from_sampling_mask(rows),
-        SamplingMask.from_sampling_mask(packed),
+    logprobs = np.array([-0.5, -1.0, -1.5], np.float32)
+    for mask_payload, logprob_field in (
+        (
+            rows,
+            {
+                "logprobs": {
+                    "content": [
+                        {"token": f"token_id:{token_id}", "logprob": float(logprob)}
+                        for token_id, logprob in zip([4, 7, 9], logprobs, strict=True)
+                    ]
+                }
+            },
+        ),
+        (
+            packed,
+            {
+                "completion_logprobs": {
+                    "data": base64.b64encode(logprobs.tobytes()).decode(),
+                    "shape": [3],
+                    "dtype": "float32",
+                }
+            },
+        ),
     ):
+        response = response_from_wire(
+            {
+                "prompt_token_ids": [1, 2],
+                "choices": [
+                    {
+                        "message": {"role": "assistant", "content": "done"},
+                        "token_ids": [4, 7, 9],
+                        "sampling_mask": mask_payload,
+                        **logprob_field,
+                    }
+                ],
+            }
+        )
+        assert response.tokens is not None
+        np.testing.assert_array_equal(response.tokens.completion_logprobs, logprobs)
+        mask = response.tokens.sampling_mask
+        assert mask is not None
         np.testing.assert_array_equal(mask.ids, ids)
         np.testing.assert_array_equal(mask.counts, counts)

verifiers/v1/__init__.py

@@ -5,13 +5,9 @@
 from verifiers.v1.acp import ACPConfig, ACPHarness, ACPTurn
 from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
 from verifiers.v1.clients import (
-    BaseClientConfig,
     Client,
     ClientConfig,
-    EvalClientConfig,
     ModelContext,
-    TrainClientConfig,
-    resolve_client,
 )
 from verifiers.v1.configs.agent import AgentConfig
 from verifiers.v1.configs.cli.env import (
@@ -287,11 +283,7 @@
     "TunnelError",
     # clients
     "Client",
-    "BaseClientConfig",
     "ClientConfig",
-    "EvalClientConfig",
-    "TrainClientConfig",
-    "resolve_client",
     # taskset / harness / runtime / environment
     "Taskset",
     "TaskConfig",

verifiers/v1/agent.py

@@ -19,7 +19,7 @@
 from typing_extensions import TypeVar
 
 from verifiers.v1.clients import (
-    EvalClientConfig,
+    ClientConfig,
     ModelContext,
 )
 from verifiers.v1.configs.agent import (
@@ -565,7 +565,7 @@ def __init__(
         self.harness = load_harness(config.harness)
         self.ctx = ModelContext(
             model=config.model,
-            client=config.client or EvalClientConfig(),
+            client=config.client or ClientConfig(),
             sampling=config.sampling,
         )
         self._closed = False

verifiers/v1/clients/__init__.py

@@ -1,21 +1,4 @@
-from verifiers.v1.clients.client import Client, ModelContext, resolve_client
-from verifiers.v1.clients.eval import EvalClient
-from verifiers.v1.clients.train import TrainClient
-from verifiers.v1.configs.client import (
-    BaseClientConfig,
-    ClientConfig,
-    EvalClientConfig,
-    TrainClientConfig,
-)
+from verifiers.v1.clients.client import Client, ModelContext
+from verifiers.v1.configs.client import ClientConfig
 
-__all__ = [
-    "BaseClientConfig",
-    "Client",
-    "ClientConfig",
-    "EvalClient",
-    "EvalClientConfig",
-    "ModelContext",
-    "TrainClient",
-    "TrainClientConfig",
-    "resolve_client",
-]
+__all__ = ["Client", "ClientConfig", "ModelContext"]

verifiers/v1/clients/base.py

@@ -6,7 +6,7 @@
 from openai import AsyncOpenAI
 
 from verifiers.v1.configs.client import (
-    BaseClientConfig,
+    ClientConfig,
     resolve_api_key,
     resolve_headers,
 )
@@ -23,13 +23,13 @@
 VERSION_SEGMENT = re.compile(r"v\d+")
 
 
-def build_async_httpx(config: BaseClientConfig) -> httpx.AsyncClient:
+def build_async_httpx(config: ClientConfig) -> httpx.AsyncClient:
     return httpx.AsyncClient(
         timeout=httpx.Timeout(**config.timeout.model_dump()), limits=DEFAULT_LIMITS
     )
 
 
-def build_async_openai(config: BaseClientConfig) -> AsyncOpenAI:
+def build_async_openai(config: ClientConfig) -> AsyncOpenAI:
     http_client = build_async_httpx(config)
     # Pass it explicitly: SDK defaults can replace a transport timeout equal to HTTPX's.
     return AsyncOpenAI(