Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
97799b3
fix: pin exclude-newer-package cutoffs as UTC timestamps
mikasenghaas Jul 29, 2026
05eb6b9
chore: align trace mutators on the record_* verb
mikasenghaas Jul 29, 2026
491b765
chore: rename trace dump exclusions to EXCLUDE_FIELDS
mikasenghaas Jul 29, 2026
df87cf1
feat: an expired agent timeout is an agent error, not a truncation
mikasenghaas Jul 29, 2026
8cb9c16
chore: slim the Trace surface
mikasenghaas Jul 29, 2026
1e48631
feat: require agent and verifiers on Trace, default tools to empty
mikasenghaas Jul 29, 2026
7963173
chore: hoist TRACE_VERSION to the module top
mikasenghaas Jul 29, 2026
2d59f06
chore: tighten trace docstrings, require RunInfo.id
mikasenghaas Jul 29, 2026
45a8bf2
fix: last trace.runtime accesses missed in the property removal
mikasenghaas Jul 29, 2026
4fca35e
feat: migrate pre-v5 trace records on read
mikasenghaas Jul 29, 2026
69e0e87
chore: restructure trace migration as chained per-version utils
mikasenghaas Jul 29, 2026
bf4a6ff
feat: retroactive trace migrations back to v1
mikasenghaas Jul 29, 2026
e520e50
chore: drop trace record migrations, reset TRACE_VERSION to 1
mikasenghaas Jul 29, 2026
b1eee4b
Merge remote-tracking branch 'origin/main' into manual-cleanups
mikasenghaas Jul 29, 2026
58e3a6a
Merge remote-tracking branch 'origin/main' into manual-cleanups
mikasenghaas Jul 29, 2026
f0723ca
fix: migrate the standalone agent example and env docs off removed tr…
mikasenghaas Jul 30, 2026
de8d521
chore: drop the seat term from new comments, restore trimmed timing d…
mikasenghaas Jul 30, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/v1/env.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class DebateEnv(vf.Env[DebateConfig]):
await agents.judge.run(VerdictTask.from_traces(task, pro, con))

async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
by_agent = {t.agent_name: t for t in episode.traces}
by_agent = {t.agent.name: t for t in episode.traces}
winner = (by_agent["judge"].last_reply or "").strip().lower()
by_agent["pro"].record_reward("won", float(winner == "pro"))
by_agent["con"].record_reward("won", float(winner == "con"))
Expand Down
4 changes: 2 additions & 2 deletions environments/proposer_solver_v1/proposer_solver_v1/taskset.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ async def run(self, task: vf.Task, agents: vf.Agents) -> None:

@staticmethod
def _solve_rate(traces: list[vf.Trace]) -> float:
solves = [t for t in traces if t.agent_name == "solver"]
solves = [t for t in traces if t.agent.name == "solver"]
if not solves:
return 0.0
return sum(
Expand All @@ -163,7 +163,7 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
the problem, 0 when it's trivial or impossible for them (4p(1-p))."""
rate = self._solve_rate(episode.traces)
for trace in episode.traces:
if trace.agent_name == "proposer":
if trace.agent.name == "proposer":
trace.record_metric("solve_rate", rate)
trace.record_reward("learnability", 4.0 * rate * (1.0 - rate))

Expand Down
5 changes: 2 additions & 3 deletions examples/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,13 @@ async def main() -> None:
async with solver:
trace = await solver.run(task)
print("stop:", trace.stop_condition)
print("error:", trace.error)
print("error:", trace.last_error)
print("turns:", trace.num_turns)
print("usage:", trace.usage)
last = trace.assistant_messages[-1].content if trace.assistant_messages else None
print("answer:", last)
assert trace.agent is not None
print("agent:", trace.agent.name, trace.agent.config.model)
print("runtime:", trace.runtime.type if trace.runtime else None)
print("runtime:", trace.agent.runtime.type if trace.agent.runtime else None)


if __name__ == "__main__":
Expand Down
6 changes: 4 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,10 @@ kuhn-poker-v1 = { path = "environments/kuhn_poker_v1", editable = true }

[tool.uv.exclude-newer-package]
# Bounded cutoffs for the explicitly requested tool upgrades.
ruff = "2026-07-27"
ty = "2026-07-27"
# Full UTC timestamps: bare dates resolve to local-tz midnight and churn the
# lockfile across timezones.
ruff = "2026-07-28T00:00:00Z"
ty = "2026-07-28T00:00:00Z"
# PrimeIntellect-published on PyPI (trusted publisher)
prime-tunnel = false
prime-sandboxes = false
Expand Down
2 changes: 1 addition & 1 deletion skills/evaluate-environments/references/REFERENCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ Trainability is not a config field: it is env truth, set in place by the env's `

The separate `verifiers/v1/configs/env.py` `TimeoutConfig` (the env's `--env.timeout.*`) keeps only `episode` — the bound on the whole `run()` interaction — and `finalize` — the bound on the env's `finalize()` hook.

> Remote sandboxes cap any harness timeout at 24 hours (provider max lifetime).
> Remote sandboxes cap any agent timeout at 24 hours (provider max lifetime).

---

Expand Down
40 changes: 20 additions & 20 deletions tests/v1/test_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
assert trace.reward == 1.0
# The interception server captured the advertised tools onto the trace (for tool-use SFT):
# the null harness offered the task's MCP tool as `echo_back`, schema included.
assert trace.tools is not None
assert trace.tools
(echo_tool,) = [t for t in trace.tools if t.name == "echo_back"]
assert "message" in echo_tool.parameters.get("properties", {})

Expand Down Expand Up @@ -355,9 +355,9 @@ async def test_multi_agent_env(run_v1, tmp_path):
max_turns=2,
)
assert len(traces) == 2 # one episode, one trace per role
assert sorted(t.agent_name for t in traces) == ["a", "b"]
(b,) = [t for t in traces if t.agent_name == "b"]
assert b.trainable is False
assert sorted(t.agent.name for t in traces) == ["a", "b"]
(b,) = [t for t in traces if t.agent.name == "b"]
assert b.agent.trainable is False
for trace in traces:
assert trace.ok
assert trace.reward == 1.0 # each seat's own task reward
Expand Down Expand Up @@ -385,7 +385,7 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
max_turns=2,
)
assert len(traces) == 2 # one episode, two attempts
assert all(t.agent_name == "agent" and t.ok for t in traces)
assert all(t.agent.name == "agent" and t.ok for t in traces)
assert any(t.metrics["best"] == 1.0 for t in traces)
assert all(t.metrics["pass_at_n"] == 1.0 for t in traces) # echo always passes

Expand Down Expand Up @@ -423,11 +423,11 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
max_turns=10,
rollout_timeout=600,
)
assert sorted(t.agent_name for t in traces) == ["judge", "solver"]
(solver,) = [t for t in traces if t.agent_name == "solver"]
(judge,) = [t for t in traces if t.agent_name == "judge"]
assert sorted(t.agent.name for t in traces) == ["judge", "solver"]
(solver,) = [t for t in traces if t.agent.name == "solver"]
(judge,) = [t for t in traces if t.agent.name == "judge"]
assert solver.ok and judge.ok
assert judge.trainable is False
assert judge.agent.trainable is False
# The task's own reward keeps its raw score; the rescale lands on the weight.
assert solver.rewards["echoed"].score == 1.0
assert solver.rewards["echoed"].weight == 0.5
Expand All @@ -448,11 +448,11 @@ async def test_env_id_user_sim(run_v1, tmp_path):
max_turns=6,
rollout_timeout=300,
)
assert sorted(t.agent_name for t in traces) == ["assistant", "user"]
(assistant,) = [t for t in traces if t.agent_name == "assistant"]
(user,) = [t for t in traces if t.agent_name == "user"]
assert sorted(t.agent.name for t in traces) == ["assistant", "user"]
(assistant,) = [t for t in traces if t.agent.name == "assistant"]
(user,) = [t for t in traces if t.agent.name == "user"]
assert assistant.ok and user.ok
assert user.trainable is False
assert user.agent.trainable is False
assert user.num_turns >= 1 # the modeled user actually spoke
assert assistant.metrics["user_turns"] >= 1
# `mask_prompt`: the scenario is hidden from the assistant's harness (the run's
Expand All @@ -465,7 +465,7 @@ async def test_env_id_user_sim(run_v1, tmp_path):
from verifiers.v1.trace import WireTrace

(record,) = read_episodes(tmp_path, WireTrace)
assert {t.agent_name for t in record.traces} == {"assistant", "user"}
assert {t.agent.name for t in record.traces} == {"assistant", "user"}
assert record.id # both traces are persisted under one durable episode identity


Expand All @@ -488,14 +488,14 @@ async def test_env_id_user_sim_with_tools(run_v1, tmp_path):
max_tokens=8192,
rollout_timeout=300,
)
(assistant,) = [t for t in traces if t.agent_name == "assistant"]
(user,) = [t for t in traces if t.agent_name == "user"]
(assistant,) = [t for t in traces if t.agent.name == "assistant"]
(user,) = [t for t in traces if t.agent.name == "user"]
assert assistant.ok and user.ok
assert assistant.task.data.prompt is None # the scenario stayed off the wire
assert user.num_turns >= 1 # the modeled user actually drove the exchange
assert assistant.rewards["echoed"].score == 1.0 # the tool ran, mid-conversation
# The tool was advertised to the masked chat exactly as to any run.
assert assistant.tools is not None
assert assistant.tools
assert any(tool.name == "echo_back" for tool in assistant.tools)


Expand All @@ -514,8 +514,8 @@ async def test_kuhn_poker_self_play(run_v1, tmp_path):
max_tokens=8192,
rollout_timeout=300,
)
assert sorted(t.agent_name for t in traces) == ["player0", "player1"]
payoffs = {t.agent_name: t.rewards["payoff"].score for t in traces}
assert sorted(t.agent.name for t in traces) == ["player0", "player1"]
payoffs = {t.agent.name: t.rewards["payoff"].score for t in traces}
assert payoffs["player0"] + payoffs["player1"] == 0 # zero-sum
assert abs(payoffs["player0"]) in (1.0, 2.0)
for trace in traces:
Expand All @@ -542,7 +542,7 @@ async def test_multi_agent_env_server(run_v1_server, tmp_path):
max_turns=2,
)
assert len(traces) == 2
assert sorted(t.agent_name for t in traces) == ["a", "b"]
assert sorted(t.agent.name for t in traces) == ["a", "b"]
for trace in traces:
assert trace.ok
assert trace.metrics["duet"] == 1.0
Expand Down
22 changes: 16 additions & 6 deletions tests/v1/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ def test_routed_experts_attributed_and_aligned_across_turns():
concatenates back to a `[tokens, layers, top_k]` array aligned 1:1 with `branch.token_ids` —
and survives the base64 wire round-trip."""
trace = vf.Trace(
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
)
user = vf.UserMessage(content="u1")
graph.prepare_turn(trace, [user]).commit(
Expand Down Expand Up @@ -94,7 +95,8 @@ def test_routed_experts_none_when_absent():
"""No routing captured (engine ran without `enable_return_routed_experts`) -> the branch
reports None and the trainer simply skips replay."""
trace = vf.Trace(
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
)
graph.prepare_turn(trace, [vf.UserMessage(content="u1")]).commit(
vf.Response(
Expand Down Expand Up @@ -128,7 +130,10 @@ def test_tool_call_hash_matches_v0_content_and_arguments_normalization():

def test_reasoning_content_participates_in_graph_prefix_matching():
task = vf.TaskData(idx=0, prompt="use a tool")
trace = vf.Trace(task=vf.TraceTask(type="Task", data=task))
trace = vf.Trace(
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=task),
)
user = vf.UserMessage(content="use a tool")
call = vf.ToolCall(id="call_0", name="lookup", arguments="{}")

Expand Down Expand Up @@ -205,7 +210,8 @@ def second_turn(trace, prompt_ids):

# Control: the prior turn re-renders to the same tokens -> stays one linear branch.
linear = vf.Trace(
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
)
first_turn(linear)
second_turn(linear, [1, 2, 3, 4, 5, 6, 7])
Expand All @@ -214,7 +220,8 @@ def second_turn(trace, prompt_ids):

# Break: the assistant turn retokenizes (4 -> 99), so prompt_ids diverge at that node.
broken = vf.Trace(
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x"))
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
)
first_turn(broken)
second_turn(broken, [1, 2, 3, 99, 5, 6, 7])
Expand All @@ -227,7 +234,10 @@ def second_turn(trace, prompt_ids):

def test_prompt_supplied_assistant_messages_are_not_sampled_turns():
task = vf.TaskData(idx=0, prompt="few-shot")
trace = vf.Trace(task=vf.TraceTask(type="Task", data=task))
trace = vf.Trace(
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=task),
)
fabricated = vf.AssistantMessage(
content=None,
tool_calls=[vf.ToolCall(id="call_0", name="lookup", arguments="{}")],
Expand Down
4 changes: 4 additions & 0 deletions tests/v1/test_judges.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ def make_trace(
task_cls: type[QAData] = QAData,
) -> vf.Trace:
return vf.Trace(
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(
type="Task",
data=task_cls(idx=0, prompt="Capital of France?", answer=answer),
Expand Down Expand Up @@ -244,6 +245,7 @@ async def test_reference_score_messages_prompt(fake_judge_model):
answer="Paris",
)
trace = vf.Trace(
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=task),
nodes=[
MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
Expand All @@ -269,6 +271,7 @@ class FieldTask(vf.TaskData):
answer="Paris",
)
trace = vf.Trace(
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=task),
nodes=[
MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
Expand All @@ -293,6 +296,7 @@ def full_trace_fixture() -> vf.Trace:
from verifiers.v1.types import ToolCall, ToolMessage

return vf.Trace(
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(
type="Task", data=QAData(idx=0, prompt="Capital of France?", answer="Paris")
),
Expand Down
5 changes: 4 additions & 1 deletion tests/v1/test_trace.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ class MyState(vf.State):
def test_bare_trace_round_trip():
# The minimal trace: a base task, no nodes, no extras — dump and back into a plain Trace.
tr = vf.Trace(
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello"))
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="Task", data=vf.TaskData(idx=3, prompt="hello")),
)
rt = vf.Trace.model_validate(tr.model_dump())
assert rt.id == tr.id
Expand All @@ -35,6 +36,7 @@ def test_custom_task_state_round_trip():
# Custom data and state round-trip into the same parameterization. Data fields are
# typed (not just `model_extra`); `state` is runtime-only and never crosses the wire.
tr = vf.Trace[MyTask, MyState](
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="gold")),
state=MyState(score=7),
nodes=[
Expand All @@ -59,6 +61,7 @@ def test_wire_trace_round_trip():
# Two leaves off one root → 2 branches (a compaction-shaped trace), so the round-trip has to
# carry node `parent` links for `num_branches` to survive.
tr = vf.Trace[MyTask, vf.State](
agent=vf.AgentInfo(config=vf.AgentConfig()),
task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="a")),
tools=[vf.Tool(name="echo", description="", parameters={"type": "object"})],
nodes=[
Expand Down
24 changes: 12 additions & 12 deletions verifiers/v1/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
UserMessage,
)
from verifiers.v1.utils.compile import (
cap_remote_harness_timeout,
cap_remote_agent_timeout,
resolve_runtime_config,
validate_pairing,
)
Expand Down Expand Up @@ -390,7 +390,7 @@ async def run(
attempt + 1,
retry.max_retries,
delay,
trace.error.type if trace.error else "?",
trace.last_error.type if trace.last_error else "?",
)
await asyncio.sleep(delay)
if history:
Expand Down Expand Up @@ -419,8 +419,8 @@ async def _run_once(
# close() never runs — free the run's servers and owned runtime first.
await run.abort()
raise
if trace.runtime is not None:
trace.runtime.borrowed = runtime is not None
if trace.agent.runtime is not None:
trace.agent.runtime.borrowed = runtime is not None
return trace

@asynccontextmanager
Expand Down Expand Up @@ -482,8 +482,8 @@ async def interaction(
opened = await run.open()
if not opened:
trace = await run.close()
if trace.runtime is not None:
trace.runtime.borrowed = runtime is not None
if trace.agent.runtime is not None:
trace.agent.runtime.borrowed = runtime is not None
if not opened:
failure = run.failure
if failure is None: # `open()` returning False always captures one.
Expand All @@ -499,8 +499,8 @@ async def interaction(
raise
finally:
trace = run.trace if run.closed else await interaction.close()
if trace.runtime is not None:
trace.runtime.borrowed = runtime is not None
if trace.agent.runtime is not None:
trace.agent.runtime.borrowed = runtime is not None

def _rollout_params(
self, task: Task, runtime: Runtime | None, shared_tools: dict
Expand All @@ -520,10 +520,10 @@ def _rollout_params(
self.harness, type(task), runtime_config, shared_tools=shared_tools
)
# Timeout precedence: agent-level wins, else the task's, else no limit.
harness_timeout = (
agent_timeout = (
self.timeout.rollout
if self.timeout.rollout is not None
else task.data.timeout.harness
else task.data.timeout.agent
)
return {
"agent_config": self.config,
Expand All @@ -535,8 +535,8 @@ def _rollout_params(
if self.timeout.setup is not None
else task.data.timeout.setup
),
"harness_timeout": cap_remote_harness_timeout(
harness_timeout, runtime_config, task
"agent_timeout": cap_remote_agent_timeout(
agent_timeout, runtime_config, task
),
"finalize_timeout": (
self.timeout.finalize
Expand Down
Loading
Loading