diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index 73f9e96e8a..a5f5135e7e 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -10,7 +10,6 @@ from verifiers.v1.cli.eval import resume from verifiers.v1.cli.output import ( append_episode, - append_trace, output_path, save_config, ) @@ -77,8 +76,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]: write_lock = asyncio.Lock() async def on_complete(episode: Episode) -> None: - for trace in episode.traces: - trace.record_run(EvalRunInfo(id=config.uuid)) + episode.record_run(EvalRunInfo(id=config.uuid)) await append_episode(out, episode, write_lock) # Serving resources (shared tool servers, interception) come up once for the @@ -258,9 +256,10 @@ async def run_group_unit(idx: int) -> list[Episode]: ) records = [] for trace in traces: - trace.record_run(EvalRunInfo(id=config.uuid)) - await append_trace(out, trace, write_lock, env=config.env_id) - records.append(Episode.of(trace)) + record = Episode.of(trace, env=config.env_id) + record.record_run(EvalRunInfo(id=config.uuid)) + await append_episode(out, record, write_lock) + records.append(record) return records async def run_unit(payload: dict) -> list[Episode]: @@ -271,8 +270,7 @@ async def run_unit(payload: dict) -> list[Episode]: sampling=config.sampling, **payload, ) - for trace in episode.traces: - trace.record_run(EvalRunInfo(id=config.uuid)) + episode.record_run(EvalRunInfo(id=config.uuid)) await append_episode(out, episode, write_lock) return [episode] diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 7e8c54bdb9..c305c1df73 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -1,14 +1,14 @@ """The episode — one run's traces plus their shared standing, whole.""" import uuid -from typing import Generic +from typing import Any, Generic from pydantic import BaseModel, Field from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData -from verifiers.v1.trace import AgentConfigT, Error, Trace +from verifiers.v1.trace import AgentConfigT, Error, RunInfo, Trace from verifiers.v1.types import Usage @@ -26,12 +26,19 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): env: EnvInfo = Field(default_factory=EnvInfo) """The env that produced this episode.""" + run: RunInfo | None = None + """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than + on each trace because the episode is what a consumer dispatches, and an episode that produced + no traces would otherwise have nowhere to say which run it was.""" ok: bool = False """Whether the episode completed successfully.""" errors: list[Error] = Field(default_factory=list) """Every error captured across attempts, oldest to newest.""" traces: list[Trace[DataT, StateT, AgentConfigT]] = Field(default_factory=list) """Every agent's trace, in completion order.""" + info: dict[str, Any] = Field(default_factory=dict) + """Scratch space for episode-level metadata, the counterpart to `Trace.info`. What describes + the whole episode belongs here rather than repeated on each of its traces.""" @property def last_error(self) -> Error | None: @@ -72,6 +79,13 @@ def by_agent(self) -> dict[str, list[Trace[DataT, StateT, AgentConfigT]]]: grouped.setdefault(trace.agent.name, []).append(trace) return grouped + def record_run(self, run: RunInfo | None = None, **info: Any) -> None: + """Record the run identity and any extra metadata about this episode. Both describe the + episode as a whole, so they are recorded once here rather than repeated on every trace.""" + if run is not None: + self.run = run + self.info.update(info) + @classmethod def of(cls, trace: Trace, env: str = "") -> "Episode": """The single-agent record: one trace as its own episode.""" diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index e88a51ba97..2e36ced9d1 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -304,9 +304,6 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """Unique ID for this trace, auto-generated.""" verifiers: VersionInfo = Field(default_factory=_current_build) """The verifiers version that produced this trace.""" - run: RunInfo | None = None - """The run this trace belongs to (eval or train), consumer-stamped.""" - task: TraceTask[DataT] """The task data that seeded this trace.""" agent: AgentInfo[AgentConfigT] @@ -480,12 +477,6 @@ def record_judge(self, response: JudgeResponse) -> None: if response.usage is not None: self.extra_usage.append(response.usage) - def record_run(self, run: RunInfo | None = None, **info: Any) -> None: - """Record the run identity (eval / train), and optional extra info.""" - if run is not None: - self.run = run - self.info.update(info) - def stop(self, condition: str) -> None: """Stop the trace, optionally with a stop condition.""" self.is_completed = True