From b705a0496a4374fe49a3a4efcebc3bd0a53ec976 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 20:42:52 +0000 Subject: [PATCH 01/19] feat(v1): name an env, and the cohort an episode was planned in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EnvInfo.name is what the caller knows the env by when that differs from its id — the key it was configured under, which a run over several envs keys its metrics by. GroupInfo is the cohort: the episodes planned together from one task, which a consumer compares against each other. Env.slots mints one per call, so -r k is a group of k, and replanning the same task later is a new group rather than a merge into the old one. Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 3 ++- verifiers/v1/env.py | 13 ++++++++++--- verifiers/v1/episode.py | 21 +++++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index f58e3dba2d..4b84b027ad 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -30,7 +30,7 @@ from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.env import Env from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig -from verifiers.v1.episode import Episode, WireEpisode +from verifiers.v1.episode import Episode, GroupInfo, WireEpisode from verifiers.v1.errors import ( EnvError, HarnessError, @@ -194,6 +194,7 @@ "WireTrace", "Reward", "Episode", + "GroupInfo", "WireEpisode", "TRACE_VERSION", "AgentInfo", diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index c7aa81db96..134de217da 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -20,7 +20,7 @@ _declared_agent_configs, default_agent_harness, ) -from verifiers.v1.episode import EnvInfo, Episode +from verifiers.v1.episode import EnvInfo, Episode, GroupInfo from verifiers.v1.errors import EnvError, boundary from verifiers.v1.harness import Harness, HarnessConfig from verifiers.v1.interception import ( @@ -56,6 +56,8 @@ class RunSlot: traces: list[Trace] = field(default_factory=list) episode: Episode | None = None done: bool = False + group: GroupInfo | None = None + """The cohort this slot was planned in, stamped onto its episode when it lands.""" @classmethod def finished(cls, episode: Episode) -> "RunSlot": @@ -64,6 +66,7 @@ def finished(cls, episode: Episode) -> "RunSlot": traces=list(episode.traces), episode=episode, done=True, + group=episode.group, ) @@ -294,10 +297,13 @@ async def run_episode( return episode def slots(self, task: Task, n: int = 1) -> list[RunSlot]: - """Plan `n` independent episodes of `task` (`-r n`): nothing couples them.""" + """Plan `n` independent episodes of `task` (`-r n`). They run independently, but they + are one cohort — the attempts a consumer compares with each other — so they share a + `GroupInfo`, which lands on each episode.""" if n < 1: raise ValueError("a task needs at least one rollout (n >= 1)") - return [RunSlot(task) for _ in range(n)] + group = GroupInfo(size=n) + return [RunSlot(task, group=group) for _ in range(n)] async def run_slot( self, @@ -330,6 +336,7 @@ def discard(trace: Trace) -> None: ) episode = await run_episode_with_retry(attempt, self.config.retries) + episode.group = slot.group slot.traces = list(episode.traces) slot.episode = episode slot.done = True diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index c305c1df73..ea632d91da 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -18,6 +18,24 @@ class EnvInfo(BaseModel): id: str = "" """`EnvConfig.env_id`, e.g. `agentic-judge+gsm8k-v1`.""" + name: str = "" + """What the caller knows this env by, when that differs from `id` — the key it was + configured under. A run over several envs keys its metrics by this; a run over one + can leave it empty.""" + + +class GroupInfo(BaseModel): + """The cohort an episode belongs to: the episodes planned together from one task, which a + consumer compares against each other (pass@k over the group, a GRPO baseline within it). + + Its `id` is per cohort, not per task: the same task planned again later is a new group, so + two rounds of it never merge into one comparison.""" + + id: str = Field(default_factory=lambda: uuid.uuid4().hex) + + size: int = 1 + """How many episodes were planned in it — the `k` of `-r k`, before any of them fail.""" + class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): """The artifact Env.run produces. Contains multiple agents' traces.""" @@ -26,6 +44,9 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): env: EnvInfo = Field(default_factory=EnvInfo) """The env that produced this episode.""" + group: GroupInfo | None = None + """The cohort it was planned in, when the producer planned one — several episodes of the + same task, meant to be compared with each other.""" run: RunInfo | None = None """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than on each trace because the episode is what a consumer dispatches, and an episode that produced From c2765a1fb84a4fa80ce66c2a9ebe5545a48f142d Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 20:51:34 +0000 Subject: [PATCH 02/19] chore(v1): trim the new docstrings, make EnvInfo.name optional Co-Authored-By: Claude Fable 5 --- verifiers/v1/env.py | 4 ++-- verifiers/v1/episode.py | 18 ++++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 134de217da..8f5cb8d43b 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -57,7 +57,7 @@ class RunSlot: episode: Episode | None = None done: bool = False group: GroupInfo | None = None - """The cohort this slot was planned in, stamped onto its episode when it lands.""" + """The group this slot was planned in, stamped onto its episode when it lands.""" @classmethod def finished(cls, episode: Episode) -> "RunSlot": @@ -298,7 +298,7 @@ async def run_episode( def slots(self, task: Task, n: int = 1) -> list[RunSlot]: """Plan `n` independent episodes of `task` (`-r n`). They run independently, but they - are one cohort — the attempts a consumer compares with each other — so they share a + are one group — the attempts a consumer compares with each other — so they share a `GroupInfo`, which lands on each episode.""" if n < 1: raise ValueError("a task needs at least one rollout (n >= 1)") diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index ea632d91da..fa803b7534 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -18,23 +18,18 @@ class EnvInfo(BaseModel): id: str = "" """`EnvConfig.env_id`, e.g. `agentic-judge+gsm8k-v1`.""" - name: str = "" - """What the caller knows this env by, when that differs from `id` — the key it was - configured under. A run over several envs keys its metrics by this; a run over one - can leave it empty.""" + name: str | None = None + """What the caller knows this env by, when that differs from `id`.""" class GroupInfo(BaseModel): - """The cohort an episode belongs to: the episodes planned together from one task, which a - consumer compares against each other (pass@k over the group, a GRPO baseline within it). - - Its `id` is per cohort, not per task: the same task planned again later is a new group, so - two rounds of it never merge into one comparison.""" + """The episodes planned together from one task, compared against each other. The `id` is + per group, not per task: planning the same task again is a new group.""" id: str = Field(default_factory=lambda: uuid.uuid4().hex) size: int = 1 - """How many episodes were planned in it — the `k` of `-r k`, before any of them fail.""" + """How many episodes were planned in it.""" class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): @@ -45,8 +40,7 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): env: EnvInfo = Field(default_factory=EnvInfo) """The env that produced this episode.""" group: GroupInfo | None = None - """The cohort it was planned in, when the producer planned one — several episodes of the - same task, meant to be compared with each other.""" + """The group it was planned in, when the producer planned one.""" run: RunInfo | None = None """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than on each trace because the episode is what a consumer dispatches, and an episode that produced From 507c4fe526c47904436f6981f1bf0a311e0b3299 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 21:04:11 +0000 Subject: [PATCH 03/19] fix(v1): a resumed run rejoins the group it left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slots() minted a group from the count it was asked to plan, but a resume asks only for what it still owes — so its replacements got a fresh id and a smaller size while the kept episodes stayed in the original group, and one -r k split into two partial groups. Group-keyed pass@k read those halves as whole groups. slots() now accepts a group to join, and the eval runner recovers each task's group from its kept episodes (resume.groups_by_key). Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/eval/resume.py | 15 ++++++++++++++- verifiers/v1/cli/eval/runner.py | 22 +++++++++++++++++----- verifiers/v1/env.py | 10 +++++++--- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/verifiers/v1/cli/eval/resume.py b/verifiers/v1/cli/eval/resume.py index 18bda68979..49db248225 100644 --- a/verifiers/v1/cli/eval/resume.py +++ b/verifiers/v1/cli/eval/resume.py @@ -23,7 +23,7 @@ from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, sniff_episode from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.episode import Episode, WireEpisode +from verifiers.v1.episode import Episode, GroupInfo, WireEpisode from verifiers.v1.trace import WireTrace K = TypeVar("K", bound=Hashable) @@ -35,6 +35,19 @@ def task_key(data: Mapping) -> str: return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest() +def groups_by_key(kept: list[Episode]) -> dict[str, GroupInfo]: + """The group each kept task is already in, by task key — what a replayed task's + replacements rejoin so a resume does not split one group in two. A task saved before + groups existed, or with none recorded, is absent.""" + groups: dict[str, GroupInfo] = {} + for episode in kept: + if episode.group is None or not episode.traces: + continue + data = episode.traces[0].task.data.model_dump(mode="json", exclude_none=True) + groups.setdefault(task_key(data), episode.group) + return groups + + def distribute( selected_keys: list[K], owed: dict[K, int], num_rollouts: int ) -> list[int]: diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index a5f5135e7e..8310c0abdb 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -16,7 +16,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.env import Env, RunSlot -from verifiers.v1.episode import Episode +from verifiers.v1.episode import Episode, GroupInfo from verifiers.v1.taskset import SEED from verifiers.v1.trace import EvalRunInfo @@ -41,8 +41,9 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]: asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None ) out = output_path(config) - # One (task, rollouts-to-run) pair per selected task; resume shrinks the counts. - plan = [(task, config.num_rollouts) for task in tasks] + # One (task, rollouts-to-run, group) triple per selected task; resume shrinks the counts + # and carries the group its kept episodes are already in. + plan = [(task, config.num_rollouts, None) for task in tasks] # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run. finished: list[Episode] = [] if config.resume is not None: @@ -55,7 +56,14 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]: print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts)) raise SystemExit(0) counts = resume.distribute(keys, owed, config.num_rollouts) - plan = [(task, n) for task, n in zip(tasks, counts) if n] + # A task's replacements rejoin the group of its kept episodes, so one `-r k` stays one + # group across the resume instead of splitting into kept and re-run halves. + kept_groups = resume.groups_by_key(finished) + plan = [ + (task, n, kept_groups.get(key) or GroupInfo(size=config.num_rollouts)) + for task, key, n in zip(tasks, keys, counts) + if n + ] logger.info( "resuming %s: %d task(s), %d rollout(s) owed", out, @@ -82,7 +90,11 @@ async def on_complete(episode: Episode) -> None: # Serving resources (shared tool servers, interception) come up once for the # run; plan slots inside so the env's agents borrow them. async with env.serving(): - planned = [slot for task, n in plan for slot in env.slots(task, n=n)] + planned = [ + slot + for task, n, group in plan + for slot in env.slots(task, n=n, group=group) + ] slots = [RunSlot.finished(episode) for episode in finished] + planned push_state = None if config.push and config.rich: diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 8f5cb8d43b..168296f4b2 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -296,13 +296,17 @@ async def run_episode( episode.ok = all(t.ok for t in episode.traces) return episode - def slots(self, task: Task, n: int = 1) -> list[RunSlot]: + def slots( + self, task: Task, n: int = 1, group: GroupInfo | None = None + ) -> list[RunSlot]: """Plan `n` independent episodes of `task` (`-r n`). They run independently, but they are one group — the attempts a consumer compares with each other — so they share a - `GroupInfo`, which lands on each episode.""" + `GroupInfo`, which lands on each episode. Pass `group` to join episodes planned + earlier: a resumed run plans only what it still owes, and its replacements belong to + the group the kept ones are already in.""" if n < 1: raise ValueError("a task needs at least one rollout (n >= 1)") - group = GroupInfo(size=n) + group = group or GroupInfo(size=n) return [RunSlot(task, group=group) for _ in range(n)] async def run_slot( From 98cdf64768351211eda8db04519473cf43615599 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 21:09:40 +0000 Subject: [PATCH 04/19] revert(v1): don't mint groups in the eval CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit group joins run as a consumer-stamped field and nothing in verifiers sets it. Minting one in slots() was speculative — no verifiers code reads it, pass@k there groups by task.data.idx — and it dragged in a resume bug: slots() sized the group by what it was asked to plan, but a resume asks only for what it owes, so one -r k split into two partial groups. Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/eval/resume.py | 15 +-------------- verifiers/v1/cli/eval/runner.py | 22 +++++----------------- verifiers/v1/env.py | 19 ++++--------------- verifiers/v1/episode.py | 2 +- 4 files changed, 11 insertions(+), 47 deletions(-) diff --git a/verifiers/v1/cli/eval/resume.py b/verifiers/v1/cli/eval/resume.py index 49db248225..18bda68979 100644 --- a/verifiers/v1/cli/eval/resume.py +++ b/verifiers/v1/cli/eval/resume.py @@ -23,7 +23,7 @@ from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, sniff_episode from verifiers.v1.configs.cli.eval import EvalConfig -from verifiers.v1.episode import Episode, GroupInfo, WireEpisode +from verifiers.v1.episode import Episode, WireEpisode from verifiers.v1.trace import WireTrace K = TypeVar("K", bound=Hashable) @@ -35,19 +35,6 @@ def task_key(data: Mapping) -> str: return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest() -def groups_by_key(kept: list[Episode]) -> dict[str, GroupInfo]: - """The group each kept task is already in, by task key — what a replayed task's - replacements rejoin so a resume does not split one group in two. A task saved before - groups existed, or with none recorded, is absent.""" - groups: dict[str, GroupInfo] = {} - for episode in kept: - if episode.group is None or not episode.traces: - continue - data = episode.traces[0].task.data.model_dump(mode="json", exclude_none=True) - groups.setdefault(task_key(data), episode.group) - return groups - - def distribute( selected_keys: list[K], owed: dict[K, int], num_rollouts: int ) -> list[int]: diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index 8310c0abdb..a5f5135e7e 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -16,7 +16,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.env import Env, RunSlot -from verifiers.v1.episode import Episode, GroupInfo +from verifiers.v1.episode import Episode from verifiers.v1.taskset import SEED from verifiers.v1.trace import EvalRunInfo @@ -41,9 +41,8 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]: asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None ) out = output_path(config) - # One (task, rollouts-to-run, group) triple per selected task; resume shrinks the counts - # and carries the group its kept episodes are already in. - plan = [(task, config.num_rollouts, None) for task in tasks] + # One (task, rollouts-to-run) pair per selected task; resume shrinks the counts. + plan = [(task, config.num_rollouts) for task in tasks] # Kept on-disk rollouts rejoin the run as finished episodes; only owed ones re-run. finished: list[Episode] = [] if config.resume is not None: @@ -56,14 +55,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]: print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts)) raise SystemExit(0) counts = resume.distribute(keys, owed, config.num_rollouts) - # A task's replacements rejoin the group of its kept episodes, so one `-r k` stays one - # group across the resume instead of splitting into kept and re-run halves. - kept_groups = resume.groups_by_key(finished) - plan = [ - (task, n, kept_groups.get(key) or GroupInfo(size=config.num_rollouts)) - for task, key, n in zip(tasks, keys, counts) - if n - ] + plan = [(task, n) for task, n in zip(tasks, counts) if n] logger.info( "resuming %s: %d task(s), %d rollout(s) owed", out, @@ -90,11 +82,7 @@ async def on_complete(episode: Episode) -> None: # Serving resources (shared tool servers, interception) come up once for the # run; plan slots inside so the env's agents borrow them. async with env.serving(): - planned = [ - slot - for task, n, group in plan - for slot in env.slots(task, n=n, group=group) - ] + planned = [slot for task, n in plan for slot in env.slots(task, n=n)] slots = [RunSlot.finished(episode) for episode in finished] + planned push_state = None if config.push and config.rich: diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 168296f4b2..c7aa81db96 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -20,7 +20,7 @@ _declared_agent_configs, default_agent_harness, ) -from verifiers.v1.episode import EnvInfo, Episode, GroupInfo +from verifiers.v1.episode import EnvInfo, Episode from verifiers.v1.errors import EnvError, boundary from verifiers.v1.harness import Harness, HarnessConfig from verifiers.v1.interception import ( @@ -56,8 +56,6 @@ class RunSlot: traces: list[Trace] = field(default_factory=list) episode: Episode | None = None done: bool = False - group: GroupInfo | None = None - """The group this slot was planned in, stamped onto its episode when it lands.""" @classmethod def finished(cls, episode: Episode) -> "RunSlot": @@ -66,7 +64,6 @@ def finished(cls, episode: Episode) -> "RunSlot": traces=list(episode.traces), episode=episode, done=True, - group=episode.group, ) @@ -296,18 +293,11 @@ async def run_episode( episode.ok = all(t.ok for t in episode.traces) return episode - def slots( - self, task: Task, n: int = 1, group: GroupInfo | None = None - ) -> list[RunSlot]: - """Plan `n` independent episodes of `task` (`-r n`). They run independently, but they - are one group — the attempts a consumer compares with each other — so they share a - `GroupInfo`, which lands on each episode. Pass `group` to join episodes planned - earlier: a resumed run plans only what it still owes, and its replacements belong to - the group the kept ones are already in.""" + def slots(self, task: Task, n: int = 1) -> list[RunSlot]: + """Plan `n` independent episodes of `task` (`-r n`): nothing couples them.""" if n < 1: raise ValueError("a task needs at least one rollout (n >= 1)") - group = group or GroupInfo(size=n) - return [RunSlot(task, group=group) for _ in range(n)] + return [RunSlot(task) for _ in range(n)] async def run_slot( self, @@ -340,7 +330,6 @@ def discard(trace: Trace) -> None: ) episode = await run_episode_with_retry(attempt, self.config.retries) - episode.group = slot.group slot.traces = list(episode.traces) slot.episode = episode slot.done = True diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index fa803b7534..85f20ae55b 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -40,7 +40,7 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): env: EnvInfo = Field(default_factory=EnvInfo) """The env that produced this episode.""" group: GroupInfo | None = None - """The group it was planned in, when the producer planned one.""" + """The group it was planned in, consumer-stamped like `run`.""" run: RunInfo | None = None """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than on each trace because the episode is what a consumer dispatches, and an episode that produced From 50f2297836c595a669b0a34e261e9e730cf99e62 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 21:20:01 +0000 Subject: [PATCH 05/19] feat(v1): the train run records the policy it came from, and Episode.to_record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit policy_version and off_policy_steps are the training path's own facts — which policy generated an episode, and how far behind the step that trains on it — so they sit on TrainRunInfo beside its id and step. to_record is the episode form of Trace.to_record: the same tensor exclusions applied through traces, which is the unit traces.jsonl stores. Co-Authored-By: Claude Fable 5 --- verifiers/v1/episode.py | 9 ++++++++- verifiers/v1/trace.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 85f20ae55b..1229531f71 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -8,7 +8,7 @@ from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData -from verifiers.v1.trace import AgentConfigT, Error, RunInfo, Trace +from verifiers.v1.trace import EXCLUDE_FIELDS, AgentConfigT, Error, RunInfo, Trace from verifiers.v1.types import Usage @@ -94,6 +94,13 @@ def by_agent(self) -> dict[str, list[Trace[DataT, StateT, AgentConfigT]]]: grouped.setdefault(trace.agent.name, []).append(trace) return grouped + def to_record(self) -> dict[str, Any]: + """JSON record without raw tensors — the episode form of `Trace.to_record`, and the unit + `traces.jsonl` stores: one episode per line.""" + return self.model_dump( + mode="json", exclude={"traces": {"__all__": EXCLUDE_FIELDS}} + ) + def record_run(self, run: RunInfo | None = None, **info: Any) -> None: """Record the run identity and any extra metadata about this episode. Both describe the episode as a whole, so they are recorded once here rather than repeated on every trace.""" diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 52e479bcbd..9ca167065d 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -145,6 +145,12 @@ class TrainRunInfo(BaseModel): id: str step: int | None = None + policy_version: int | None = None + """The version of the policy that generated the episode.""" + + off_policy_steps: int = 0 + """How many policy versions behind the step training on it — 0 when on-policy.""" + RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] """The run a trace belongs to, discriminated on `type`.""" From d3be61d6781cbb90e587bd35689c92bd8e82a048 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 21:22:36 +0000 Subject: [PATCH 06/19] feat(v1): an eval run records the policy version it measures Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 9ca167065d..1352fd8cc8 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -138,6 +138,9 @@ class EvalRunInfo(BaseModel): id: str step: int | None = None + policy_version: int | None = None + """The version of the policy this measures.""" + class TrainRunInfo(BaseModel): type: Literal["train"] = "train" @@ -146,10 +149,11 @@ class TrainRunInfo(BaseModel): step: int | None = None policy_version: int | None = None - """The version of the policy that generated the episode.""" + """The version of the policy this was generated from.""" off_policy_steps: int = 0 - """How many policy versions behind the step training on it — 0 when on-policy.""" + """How many versions behind the step training on it — 0 when on-policy. Only the training + path goes stale; an eval is measured against the policy it ran on.""" RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] From c12e77bf21d03026ad14a89f783587fb795b1790 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 21:24:42 +0000 Subject: [PATCH 07/19] feat(v1): online eval belongs to the training run A training run's evals are its own: same run id, same step, same policy as the rollouts it trains on. They were an EvalRunInfo, which made those shared facts optional on both records and left a standalone eval carrying a step and a policy version it has none of. TrainRunInfo now covers both through kind, and EvalRunInfo is what it should always have been: a model measured once, against nothing that is training. Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 1352fd8cc8..8bc9fd1d71 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -133,27 +133,35 @@ def value(self) -> float: class EvalRunInfo(BaseModel): + """A standalone eval: a model measured once, against nothing that is training.""" + type: Literal["eval"] = "eval" id: str - step: int | None = None - - policy_version: int | None = None - """The version of the policy this measures.""" class TrainRunInfo(BaseModel): + """A training run. Its episodes are the ones it trains on and the ones it evaluates along + the way — both belong to the run and both are placed by the same step and policy, so they + are one record distinguished by `kind` rather than two.""" + type: Literal["train"] = "train" id: str + + kind: Literal["train", "eval"] = "train" + """Whether the run trains on this episode or only measures it.""" + step: int | None = None + """The step it belongs to: the one whose eval produced it, or — for an episode trained on — + the batch window collecting when it lands, which is not known until it does.""" policy_version: int | None = None - """The version of the policy this was generated from.""" + """The version of the policy that produced it.""" off_policy_steps: int = 0 - """How many versions behind the step training on it — 0 when on-policy. Only the training - path goes stale; an eval is measured against the policy it ran on.""" + """How many versions behind the step training on it. Always 0 for `kind="eval"`, which is + measured against the policy it ran on.""" RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] From 70490aa3efe7080e1a4b138da58eb63b295b1771 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 21:27:22 +0000 Subject: [PATCH 08/19] chore(v1): export EnvInfo Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 4b84b027ad..177887c552 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -30,7 +30,7 @@ from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.env import Env from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig -from verifiers.v1.episode import Episode, GroupInfo, WireEpisode +from verifiers.v1.episode import EnvInfo, Episode, GroupInfo, WireEpisode from verifiers.v1.errors import ( EnvError, HarnessError, @@ -194,6 +194,7 @@ "WireTrace", "Reward", "Episode", + "EnvInfo", "GroupInfo", "WireEpisode", "TRACE_VERSION", From 8c318d0a4bc9e8a145267b96701cf02bebb7fc01 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 21:55:01 +0000 Subject: [PATCH 09/19] feat(v1): a run records the policy span, and derives staleness from it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staleness was a number a consumer had to compute and store, and it only ever described one end of the episode. The run now records the policy versions generation spanned — TimeSpan's shape over updates instead of seconds — and both readings fall out: drift is what changed mid-episode, off_policy_steps how far behind training the generating policy was. Neither is stored, so neither can disagree with the span. A frozen sampler follows no version, so its policy is None and staleness with it. Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 2 ++ verifiers/v1/trace.py | 34 +++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 177887c552..e1797f4013 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -77,6 +77,7 @@ Error, EvalRunInfo, ModelCall, + PolicySpan, Reward, RunInfo, TimeSpan, @@ -201,6 +202,7 @@ "AgentInfo", "RunInfo", "EvalRunInfo", + "PolicySpan", "ModelCall", "TrainRunInfo", "VersionInfo", diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 8bc9fd1d71..1f0882c6ea 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -140,6 +140,23 @@ class EvalRunInfo(BaseModel): id: str +class PolicySpan(BaseModel): + """The live policy versions an episode was generated across — `TimeSpan`'s shape, measured in + policy updates instead of seconds.""" + + start: int = 0 + """The version generation began under: the policy that produced most of the episode.""" + + end: int = 0 + """The version in force when it finished.""" + + @property + def drift(self) -> int: + """Updates that landed mid-generation. 0 when one policy produced the whole episode; above + that, the episode is a blend and its later turns came from a newer policy than its first.""" + return max(0, self.end - self.start) + + class TrainRunInfo(BaseModel): """A training run. Its episodes are the ones it trains on and the ones it evaluates along the way — both belong to the run and both are placed by the same step and policy, so they @@ -156,12 +173,19 @@ class TrainRunInfo(BaseModel): """The step it belongs to: the one whose eval produced it, or — for an episode trained on — the batch window collecting when it lands, which is not known until it does.""" - policy_version: int | None = None - """The version of the policy that produced it.""" + policy: PolicySpan | None = None + """The live policy versions it was generated across. `None` when it was not generated from the + live policy at all — a frozen sampler follows no version, so staleness has no meaning for it.""" - off_policy_steps: int = 0 - """How many versions behind the step training on it. Always 0 for `kind="eval"`, which is - measured against the policy it ran on.""" + @property + def off_policy_steps(self) -> int | None: + """How many versions behind the policy in training the generating policy was, or `None` + when there is nothing to compare. A run's step `n` trains the policy that step `n-1` + produced, so an episode generated under `v{k}` and placed in step `n` is `(n-1)-k` behind — + queue time included, since `step` is the window it landed in, not the one it left.""" + if self.policy is None or self.step is None: + return None + return max(0, (self.step - 1) - self.policy.start) RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] From def0d5af1fd06d68026fca0350bd66059a63a6c5 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 22:12:01 +0000 Subject: [PATCH 10/19] chore(v1): drop Episode.to_record, plainer wording on the policy span MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit to_record had no caller here — write_episode dumps the episode itself, and the eval path has no tensors to exclude. It was added for a consumer that can hold it. Co-Authored-By: Claude Fable 5 --- verifiers/v1/episode.py | 9 +-------- verifiers/v1/trace.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 1229531f71..85f20ae55b 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -8,7 +8,7 @@ from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData -from verifiers.v1.trace import EXCLUDE_FIELDS, AgentConfigT, Error, RunInfo, Trace +from verifiers.v1.trace import AgentConfigT, Error, RunInfo, Trace from verifiers.v1.types import Usage @@ -94,13 +94,6 @@ def by_agent(self) -> dict[str, list[Trace[DataT, StateT, AgentConfigT]]]: grouped.setdefault(trace.agent.name, []).append(trace) return grouped - def to_record(self) -> dict[str, Any]: - """JSON record without raw tensors — the episode form of `Trace.to_record`, and the unit - `traces.jsonl` stores: one episode per line.""" - return self.model_dump( - mode="json", exclude={"traces": {"__all__": EXCLUDE_FIELDS}} - ) - def record_run(self, run: RunInfo | None = None, **info: Any) -> None: """Record the run identity and any extra metadata about this episode. Both describe the episode as a whole, so they are recorded once here rather than repeated on every trace.""" diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 1f0882c6ea..5e1a3973d2 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -142,18 +142,18 @@ class EvalRunInfo(BaseModel): class PolicySpan(BaseModel): """The live policy versions an episode was generated across — `TimeSpan`'s shape, measured in - policy updates instead of seconds.""" + policy updates.""" start: int = 0 - """The version generation began under: the policy that produced most of the episode.""" + """The live version when generation began.""" end: int = 0 - """The version in force when it finished.""" + """The live version when it finished.""" @property def drift(self) -> int: - """Updates that landed mid-generation. 0 when one policy produced the whole episode; above - that, the episode is a blend and its later turns came from a newer policy than its first.""" + """Updates that landed mid-generation: above 0, the episode's later turns came from a + newer policy than its first.""" return max(0, self.end - self.start) @@ -179,10 +179,10 @@ class TrainRunInfo(BaseModel): @property def off_policy_steps(self) -> int | None: - """How many versions behind the policy in training the generating policy was, or `None` - when there is nothing to compare. A run's step `n` trains the policy that step `n-1` - produced, so an episode generated under `v{k}` and placed in step `n` is `(n-1)-k` behind — - queue time included, since `step` is the window it landed in, not the one it left.""" + """How far behind the policy being trained the generating policy was, or `None` when + there is nothing to compare. Step `n` trains the policy step `n-1` produced, so an episode + generated at `v{k}` and placed in step `n` is `(n-1)-k` behind — queue time included, since + `step` is the window it landed in, not the one it left.""" if self.policy is None or self.step is None: return None return max(0, (self.step - 1) - self.policy.start) From 54eef2cf9715418cfd1054c54d6d3803cd955c25 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 22:13:15 +0000 Subject: [PATCH 11/19] docs(v1): say what off_policy_steps measures for each kind Queue time is only in it on the training path, where step is the window the episode landed in. An eval is placed by the epoch it was dispatched for, so its lag is the one it had when it started. Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 5e1a3973d2..8949ca3cb4 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -179,10 +179,15 @@ class TrainRunInfo(BaseModel): @property def off_policy_steps(self) -> int | None: - """How far behind the policy being trained the generating policy was, or `None` when - there is nothing to compare. Step `n` trains the policy step `n-1` produced, so an episode - generated at `v{k}` and placed in step `n` is `(n-1)-k` behind — queue time included, since - `step` is the window it landed in, not the one it left.""" + """How far behind the policy being trained at `step` the generating policy was, or `None` + when there is nothing to compare. Step `n` trains the policy step `n-1` produced, so an + episode generated at `v{k}` is `(n-1)-k` behind. + + What that measures follows what `step` means for the kind. An episode trained on is placed + by the window it landed in, so its lag includes the time it sat in the queue. An eval is + placed by the epoch it was dispatched for, so its lag is the one it had when it started — + a slow eval that outlives several updates still reports that, because what it measured is + the policy it ran, not the one that has since replaced it.""" if self.policy is None or self.step is None: return None return max(0, (self.step - 1) - self.policy.start) From 260b6f1426bb510d01d7a2441a5e389c30fbe224 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 22:22:06 +0000 Subject: [PATCH 12/19] feat(v1)!: what an episode is to its run is a nested, discriminated metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kind was a field, and step meant one thing under it and another under the other value: the window an episode landed in, or the epoch an eval was dispatched for. Two meanings, one field, and only one of them could be required. They become TrainMetadata and EvalMetadata, nested under the run because that is the relationship — one training run, one id, two things an episode can be to it. Each states its own step, and an eval's is required, since it is known the moment the eval is dispatched. Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 4 ++++ verifiers/v1/trace.py | 52 ++++++++++++++++++++++++---------------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index e1797f4013..4c6b735113 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -75,6 +75,7 @@ AgentSpan, Branch, Error, + EvalMetadata, EvalRunInfo, ModelCall, PolicySpan, @@ -85,6 +86,7 @@ Timing, Trace, TraceTask, + TrainMetadata, TrainRunInfo, VersionInfo, WireTrace, @@ -202,6 +204,8 @@ "AgentInfo", "RunInfo", "EvalRunInfo", + "EvalMetadata", + "TrainMetadata", "PolicySpan", "ModelCall", "TrainRunInfo", diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 8949ca3cb4..bac0a0c831 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -157,21 +157,37 @@ def drift(self) -> int: return max(0, self.end - self.start) +class TrainMetadata(BaseModel): + """An episode a training run trains on.""" + + type: Literal["train"] = "train" + + step: int | None = None + """The batch window collecting when it landed, which is not known until it does.""" + + +class EvalMetadata(BaseModel): + """An episode a training run measures itself with.""" + + type: Literal["eval"] = "eval" + + step: int + """The step whose eval produced it, known from the moment it is dispatched.""" + + +EpisodeMetadata = Annotated[TrainMetadata | EvalMetadata, Field(discriminator="type")] +"""What an episode is to the training run it belongs to.""" + + class TrainRunInfo(BaseModel): - """A training run. Its episodes are the ones it trains on and the ones it evaluates along - the way — both belong to the run and both are placed by the same step and policy, so they - are one record distinguished by `kind` rather than two.""" + """A training run. Its episodes are the ones it trains on and the ones it evaluates along the + way: one run, one id, and `metadata` says which of the two an episode is.""" type: Literal["train"] = "train" id: str - kind: Literal["train", "eval"] = "train" - """Whether the run trains on this episode or only measures it.""" - - step: int | None = None - """The step it belongs to: the one whose eval produced it, or — for an episode trained on — - the batch window collecting when it lands, which is not known until it does.""" + metadata: EpisodeMetadata = Field(default_factory=TrainMetadata) policy: PolicySpan | None = None """The live policy versions it was generated across. `None` when it was not generated from the @@ -179,18 +195,14 @@ class TrainRunInfo(BaseModel): @property def off_policy_steps(self) -> int | None: - """How far behind the policy being trained at `step` the generating policy was, or `None` - when there is nothing to compare. Step `n` trains the policy step `n-1` produced, so an - episode generated at `v{k}` is `(n-1)-k` behind. - - What that measures follows what `step` means for the kind. An episode trained on is placed - by the window it landed in, so its lag includes the time it sat in the queue. An eval is - placed by the epoch it was dispatched for, so its lag is the one it had when it started — - a slow eval that outlives several updates still reports that, because what it measured is - the policy it ran, not the one that has since replaced it.""" - if self.policy is None or self.step is None: + """How far behind the policy being trained at `metadata.step` the generating policy was, or + `None` when there is nothing to compare. Step `n` trains the policy step `n-1` produced, so + an episode generated at `v{k}` is `(n-1)-k` behind — measured against whichever step its + metadata places it in.""" + step = self.metadata.step + if self.policy is None or step is None: return None - return max(0, (self.step - 1) - self.policy.start) + return max(0, (step - 1) - self.policy.start) RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] From 66dccbfd28fa65b1078a308f4a6c49abe1dfcc98 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 22:56:26 +0000 Subject: [PATCH 13/19] chore(v1): drop the GroupInfo docstring Co-Authored-By: Claude Fable 5 --- verifiers/v1/episode.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 85f20ae55b..1b3d526e67 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -23,9 +23,6 @@ class EnvInfo(BaseModel): class GroupInfo(BaseModel): - """The episodes planned together from one task, compared against each other. The `id` is - per group, not per task: planning the same task again is a new group.""" - id: str = Field(default_factory=lambda: uuid.uuid4().hex) size: int = 1 From 9576a5a9e315e28e5154c1fdca1dd577df971380 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 22:57:49 +0000 Subject: [PATCH 14/19] fix(v1): an eval's off-policy is its drift, not its distance from a step An eval's step is fixed when it is dispatched, so measuring staleness against it undercounts: an eval that outlives three updates still looked on-policy, while one that measured a single version cleanly reported the gap between the step index and that version. Nothing trains on an eval, so there is no policy for it to be behind. What makes it off-policy is the policy moving under it. Each metadata now owns that definition, which is what splitting them was for. Co-Authored-By: Claude Fable 5 --- verifiers/v1/trace.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index bac0a0c831..a9f4241831 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -165,6 +165,14 @@ class TrainMetadata(BaseModel): step: int | None = None """The batch window collecting when it landed, which is not known until it does.""" + def off_policy_steps(self, policy: PolicySpan) -> int | None: + """How far behind the policy being trained the generating policy was. Step `n` trains the + policy step `n-1` produced, so an episode generated at `v{k}` is `(n-1)-k` behind — queue + time included, since `step` is the window it landed in, not the one it left.""" + if self.step is None: + return None + return max(0, (self.step - 1) - policy.start) + class EvalMetadata(BaseModel): """An episode a training run measures itself with.""" @@ -174,6 +182,15 @@ class EvalMetadata(BaseModel): step: int """The step whose eval produced it, known from the moment it is dispatched.""" + def off_policy_steps(self, policy: PolicySpan) -> int: + """How far off the policy it measured drifted while it ran. Nothing trains on an eval, so + there is no policy for it to be behind; what makes it off-policy is the policy moving under + it, leaving it a measurement of a blend rather than of the version it started on. + + Its `step` cannot answer that — it is fixed when the eval is dispatched, so a slow eval that + outlives several updates would still look on-policy.""" + return policy.drift + EpisodeMetadata = Annotated[TrainMetadata | EvalMetadata, Field(discriminator="type")] """What an episode is to the training run it belongs to.""" @@ -195,14 +212,11 @@ class TrainRunInfo(BaseModel): @property def off_policy_steps(self) -> int | None: - """How far behind the policy being trained at `metadata.step` the generating policy was, or - `None` when there is nothing to compare. Step `n` trains the policy step `n-1` produced, so - an episode generated at `v{k}` is `(n-1)-k` behind — measured against whichever step its - metadata places it in.""" - step = self.metadata.step - if self.policy is None or step is None: + """How far off-policy this episode is, as its metadata defines that, or `None` when there + is no span to measure against.""" + if self.policy is None: return None - return max(0, (step - 1) - self.policy.start) + return self.metadata.off_policy_steps(self.policy) RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] From eddc781f22174ebefb749d0babf2367fdd034af3 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 23:05:53 +0000 Subject: [PATCH 15/19] refactor(v1): the run types live with the episode they describe They were written when the run was a field on Trace. #2244 moved the field to Episode; the types stayed behind, and Trace has not referred to one since. Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 19 ++++--- verifiers/v1/cli/eval/runner.py | 3 +- verifiers/v1/episode.py | 89 ++++++++++++++++++++++++++++++- verifiers/v1/trace.py | 93 +-------------------------------- 4 files changed, 101 insertions(+), 103 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 4c6b735113..a81c27250d 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -30,7 +30,18 @@ from verifiers.v1.configs.taskset import TasksetConfig from verifiers.v1.env import Env from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig -from verifiers.v1.episode import EnvInfo, Episode, GroupInfo, WireEpisode +from verifiers.v1.episode import ( + EnvInfo, + Episode, + EvalMetadata, + EvalRunInfo, + GroupInfo, + PolicySpan, + RunInfo, + TrainMetadata, + TrainRunInfo, + WireEpisode, +) from verifiers.v1.errors import ( EnvError, HarnessError, @@ -75,19 +86,13 @@ AgentSpan, Branch, Error, - EvalMetadata, - EvalRunInfo, ModelCall, - PolicySpan, Reward, - RunInfo, TimeSpan, TimeSplit, Timing, Trace, TraceTask, - TrainMetadata, - TrainRunInfo, VersionInfo, WireTrace, ) diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py index a5f5135e7e..ec9115cfd1 100644 --- a/verifiers/v1/cli/eval/runner.py +++ b/verifiers/v1/cli/eval/runner.py @@ -16,9 +16,8 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.env import Env, RunSlot -from verifiers.v1.episode import Episode +from verifiers.v1.episode import Episode, EvalRunInfo from verifiers.v1.taskset import SEED -from verifiers.v1.trace import EvalRunInfo logger = logging.getLogger(__name__) diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 1b3d526e67..a15aa9a96f 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -1,14 +1,14 @@ """The episode — one run's traces plus their shared standing, whole.""" import uuid -from typing import Any, Generic +from typing import Annotated, Any, Generic, Literal from pydantic import BaseModel, Field from verifiers.v1.configs.agent import WireAgentConfig from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData -from verifiers.v1.trace import AgentConfigT, Error, RunInfo, Trace +from verifiers.v1.trace import AgentConfigT, Error, Trace from verifiers.v1.types import Usage @@ -29,6 +29,91 @@ class GroupInfo(BaseModel): """How many episodes were planned in it.""" +class EvalRunInfo(BaseModel): + """A standalone eval: a model measured once, against nothing that is training.""" + + type: Literal["eval"] = "eval" + + id: str + + +class PolicySpan(BaseModel): + """The live policy versions an episode was generated across — `TimeSpan`'s shape, measured in + policy updates.""" + + start: int = 0 + """The live version when generation began.""" + + end: int = 0 + """The live version when it finished.""" + + @property + def drift(self) -> int: + return max(0, self.end - self.start) + + +class TrainMetadata(BaseModel): + """An episode a training run trains on.""" + + type: Literal["train"] = "train" + + step: int | None = None + """The batch window collecting when it landed, which is not known until it does.""" + + def off_policy_steps(self, policy: PolicySpan) -> int | None: + """How far behind the policy being trained the generating policy was — queue time + included, since `step` is the window it landed in, not the one it left.""" + if self.step is None: + return None + return max(0, (self.step - 1) - policy.start) + + +class EvalMetadata(BaseModel): + """An episode a training run measures itself with.""" + + type: Literal["eval"] = "eval" + + step: int + """The step whose eval produced it, known from the moment it is dispatched.""" + + def off_policy_steps(self, policy: PolicySpan) -> int: + """How far off the policy it measured drifted while it ran. Nothing trains on an eval, so + there is no policy for it to be behind; what makes it off-policy is the policy moving under + it, leaving it a measurement of a blend rather than of the version it started on. + + Its `step` cannot answer that — it is fixed when the eval is dispatched, so a slow eval that + outlives several updates would still look on-policy.""" + return policy.drift + + +EpisodeMetadata = Annotated[TrainMetadata | EvalMetadata, Field(discriminator="type")] +"""What an episode is to the training run it belongs to.""" + + +class TrainRunInfo(BaseModel): + """A training run. Its episodes are the ones it trains on and the ones it evaluates along the + way: one run, one id, and `metadata` says which of the two an episode is.""" + + type: Literal["train"] = "train" + + id: str + + metadata: EpisodeMetadata = Field(default_factory=TrainMetadata) + + policy: PolicySpan | None = None + """`None` when it was not generated from the live policy at all, as with a frozen sampler.""" + + @property + def off_policy_steps(self) -> int | None: + if self.policy is None: + return None + return self.metadata.off_policy_steps(self.policy) + + +RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] +"""The run an episode belongs to, discriminated on `type`.""" + + class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): """The artifact Env.run produces. Contains multiple agents' traces.""" diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index a9f4241831..e5853313e8 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -4,7 +4,7 @@ import traceback import uuid from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Annotated, Any, Generic, Literal +from typing import TYPE_CHECKING, Any, Generic import numpy as np from pydantic import BaseModel, Field, PrivateAttr @@ -132,97 +132,6 @@ def value(self) -> float: return self.score * self.weight -class EvalRunInfo(BaseModel): - """A standalone eval: a model measured once, against nothing that is training.""" - - type: Literal["eval"] = "eval" - - id: str - - -class PolicySpan(BaseModel): - """The live policy versions an episode was generated across — `TimeSpan`'s shape, measured in - policy updates.""" - - start: int = 0 - """The live version when generation began.""" - - end: int = 0 - """The live version when it finished.""" - - @property - def drift(self) -> int: - """Updates that landed mid-generation: above 0, the episode's later turns came from a - newer policy than its first.""" - return max(0, self.end - self.start) - - -class TrainMetadata(BaseModel): - """An episode a training run trains on.""" - - type: Literal["train"] = "train" - - step: int | None = None - """The batch window collecting when it landed, which is not known until it does.""" - - def off_policy_steps(self, policy: PolicySpan) -> int | None: - """How far behind the policy being trained the generating policy was. Step `n` trains the - policy step `n-1` produced, so an episode generated at `v{k}` is `(n-1)-k` behind — queue - time included, since `step` is the window it landed in, not the one it left.""" - if self.step is None: - return None - return max(0, (self.step - 1) - policy.start) - - -class EvalMetadata(BaseModel): - """An episode a training run measures itself with.""" - - type: Literal["eval"] = "eval" - - step: int - """The step whose eval produced it, known from the moment it is dispatched.""" - - def off_policy_steps(self, policy: PolicySpan) -> int: - """How far off the policy it measured drifted while it ran. Nothing trains on an eval, so - there is no policy for it to be behind; what makes it off-policy is the policy moving under - it, leaving it a measurement of a blend rather than of the version it started on. - - Its `step` cannot answer that — it is fixed when the eval is dispatched, so a slow eval that - outlives several updates would still look on-policy.""" - return policy.drift - - -EpisodeMetadata = Annotated[TrainMetadata | EvalMetadata, Field(discriminator="type")] -"""What an episode is to the training run it belongs to.""" - - -class TrainRunInfo(BaseModel): - """A training run. Its episodes are the ones it trains on and the ones it evaluates along the - way: one run, one id, and `metadata` says which of the two an episode is.""" - - type: Literal["train"] = "train" - - id: str - - metadata: EpisodeMetadata = Field(default_factory=TrainMetadata) - - policy: PolicySpan | None = None - """The live policy versions it was generated across. `None` when it was not generated from the - live policy at all — a frozen sampler follows no version, so staleness has no meaning for it.""" - - @property - def off_policy_steps(self) -> int | None: - """How far off-policy this episode is, as its metadata defines that, or `None` when there - is no span to measure against.""" - if self.policy is None: - return None - return self.metadata.off_policy_steps(self.policy) - - -RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] -"""The run a trace belongs to, discriminated on `type`.""" - - class ModelCall(BaseModel): """A model call, automatically recorded at intercept time.""" From a79e674459be12ae26d6f5886e7a021141966269 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 4 Aug 2026 23:35:30 +0000 Subject: [PATCH 16/19] chore(v1): docstrings in the file's own register Co-Authored-By: Claude Fable 5 --- verifiers/v1/episode.py | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index a15aa9a96f..b272e4dc42 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -23,29 +23,24 @@ class EnvInfo(BaseModel): class GroupInfo(BaseModel): - id: str = Field(default_factory=lambda: uuid.uuid4().hex) + """The episodes planned together from one task, compared against each other.""" + id: str = Field(default_factory=lambda: uuid.uuid4().hex) size: int = 1 - """How many episodes were planned in it.""" class EvalRunInfo(BaseModel): - """A standalone eval: a model measured once, against nothing that is training.""" + """A standalone eval: a model measured against nothing that is training.""" type: Literal["eval"] = "eval" - id: str class PolicySpan(BaseModel): - """The live policy versions an episode was generated across — `TimeSpan`'s shape, measured in - policy updates.""" + """Live policy versions with a derived, non-serialized drift in updates.""" start: int = 0 - """The live version when generation began.""" - end: int = 0 - """The live version when it finished.""" @property def drift(self) -> int: @@ -56,13 +51,11 @@ class TrainMetadata(BaseModel): """An episode a training run trains on.""" type: Literal["train"] = "train" - step: int | None = None - """The batch window collecting when it landed, which is not known until it does.""" + """The batch window it landed in, which is not known until it does.""" def off_policy_steps(self, policy: PolicySpan) -> int | None: - """How far behind the policy being trained the generating policy was — queue time - included, since `step` is the window it landed in, not the one it left.""" + """Versions behind the policy in training, queue time included.""" if self.step is None: return None return max(0, (self.step - 1) - policy.start) @@ -72,17 +65,12 @@ class EvalMetadata(BaseModel): """An episode a training run measures itself with.""" type: Literal["eval"] = "eval" - step: int - """The step whose eval produced it, known from the moment it is dispatched.""" + """The step whose eval produced it, known when it is dispatched.""" def off_policy_steps(self, policy: PolicySpan) -> int: - """How far off the policy it measured drifted while it ran. Nothing trains on an eval, so - there is no policy for it to be behind; what makes it off-policy is the policy moving under - it, leaving it a measurement of a blend rather than of the version it started on. - - Its `step` cannot answer that — it is fixed when the eval is dispatched, so a slow eval that - outlives several updates would still look on-policy.""" + """Versions the policy moved under it. Nothing trains on an eval, so it can only be + off-policy by drifting — and its `step` is fixed at dispatch, so it cannot say that.""" return policy.drift @@ -91,17 +79,14 @@ def off_policy_steps(self, policy: PolicySpan) -> int: class TrainRunInfo(BaseModel): - """A training run. Its episodes are the ones it trains on and the ones it evaluates along the - way: one run, one id, and `metadata` says which of the two an episode is.""" + """A training run: one id over the episodes it trains on and the ones it evaluates itself with, + which `metadata` tells apart.""" type: Literal["train"] = "train" - id: str - metadata: EpisodeMetadata = Field(default_factory=TrainMetadata) - policy: PolicySpan | None = None - """`None` when it was not generated from the live policy at all, as with a frozen sampler.""" + """`None` when it was not generated from the live policy, as with a frozen sampler.""" @property def off_policy_steps(self) -> int | None: From 9313f4b88e15e566db3dfb90254ef8da92f54b7f Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 5 Aug 2026 00:35:56 +0000 Subject: [PATCH 17/19] refactor(v1)!: the policy span belongs to the episode's metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It sat beside the run id, where it was the one per-episode fact among shared ones — the same mixing that nesting the metadata was meant to undo. Moving it completes that: a run is an id, and everything true of a single episode is its metadata. off_policy_steps follows it, so each kind owns both the definition and the data it reads, and it is a property rather than a method the run has to hand the span to. Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 2 ++ verifiers/v1/episode.py | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index a81c27250d..b4a5daeb46 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -36,6 +36,7 @@ EvalMetadata, EvalRunInfo, GroupInfo, + Metadata, PolicySpan, RunInfo, TrainMetadata, @@ -210,6 +211,7 @@ "RunInfo", "EvalRunInfo", "EvalMetadata", + "Metadata", "TrainMetadata", "PolicySpan", "ModelCall", diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index b272e4dc42..11838b6a3d 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -47,35 +47,43 @@ def drift(self) -> int: return max(0, self.end - self.start) -class TrainMetadata(BaseModel): +class Metadata(BaseModel): + """What one episode is to the training run it belongs to.""" + + policy: PolicySpan | None = None + """`None` when it was not generated from the live policy, as with a frozen sampler.""" + + +class TrainMetadata(Metadata): """An episode a training run trains on.""" type: Literal["train"] = "train" step: int | None = None """The batch window it landed in, which is not known until it does.""" - def off_policy_steps(self, policy: PolicySpan) -> int | None: + @property + def off_policy_steps(self) -> int | None: """Versions behind the policy in training, queue time included.""" - if self.step is None: + if self.policy is None or self.step is None: return None - return max(0, (self.step - 1) - policy.start) + return max(0, (self.step - 1) - self.policy.start) -class EvalMetadata(BaseModel): +class EvalMetadata(Metadata): """An episode a training run measures itself with.""" type: Literal["eval"] = "eval" step: int """The step whose eval produced it, known when it is dispatched.""" - def off_policy_steps(self, policy: PolicySpan) -> int: + @property + def off_policy_steps(self) -> int | None: """Versions the policy moved under it. Nothing trains on an eval, so it can only be off-policy by drifting — and its `step` is fixed at dispatch, so it cannot say that.""" - return policy.drift + return self.policy.drift if self.policy else None EpisodeMetadata = Annotated[TrainMetadata | EvalMetadata, Field(discriminator="type")] -"""What an episode is to the training run it belongs to.""" class TrainRunInfo(BaseModel): @@ -85,14 +93,6 @@ class TrainRunInfo(BaseModel): type: Literal["train"] = "train" id: str metadata: EpisodeMetadata = Field(default_factory=TrainMetadata) - policy: PolicySpan | None = None - """`None` when it was not generated from the live policy, as with a frozen sampler.""" - - @property - def off_policy_steps(self) -> int | None: - if self.policy is None: - return None - return self.metadata.off_policy_steps(self.policy) RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")] From 3a38df548e27ef58c3a2e7b151fc7962a64f8d8a Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 5 Aug 2026 21:52:58 +0000 Subject: [PATCH 18/19] revert(v1): drop GroupInfo from the episode Nothing here reads a group, so it was a field verifiers carried for one consumer. It belongs where it is used until something here uses it. Co-Authored-By: Claude Fable 5 --- verifiers/v1/__init__.py | 2 -- verifiers/v1/episode.py | 9 --------- 2 files changed, 11 deletions(-) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index b4a5daeb46..078c4987a3 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -35,7 +35,6 @@ Episode, EvalMetadata, EvalRunInfo, - GroupInfo, Metadata, PolicySpan, RunInfo, @@ -204,7 +203,6 @@ "Reward", "Episode", "EnvInfo", - "GroupInfo", "WireEpisode", "TRACE_VERSION", "AgentInfo", diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py index 11838b6a3d..a596f211e9 100644 --- a/verifiers/v1/episode.py +++ b/verifiers/v1/episode.py @@ -22,13 +22,6 @@ class EnvInfo(BaseModel): """What the caller knows this env by, when that differs from `id`.""" -class GroupInfo(BaseModel): - """The episodes planned together from one task, compared against each other.""" - - id: str = Field(default_factory=lambda: uuid.uuid4().hex) - size: int = 1 - - class EvalRunInfo(BaseModel): """A standalone eval: a model measured against nothing that is training.""" @@ -106,8 +99,6 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]): env: EnvInfo = Field(default_factory=EnvInfo) """The env that produced this episode.""" - group: GroupInfo | None = None - """The group it was planned in, consumer-stamped like `run`.""" run: RunInfo | None = None """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than on each trace because the episode is what a consumer dispatches, and an episode that produced From 306a95fc0ede876f17889a28c36755d2a922d271 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 5 Aug 2026 22:43:49 +0000 Subject: [PATCH 19/19] fix(v1): keep raw node tensors out of the written episode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_episode dumped the episode whole, so a producer whose nodes carry multi_modal_data or routed_experts wrote them into traces.jsonl — the exclusions Trace.to_record defines were never applied on the episode path. Nothing here populates those fields, so it was latent. Co-Authored-By: Claude Fable 5 --- verifiers/v1/cli/output.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/verifiers/v1/cli/output.py b/verifiers/v1/cli/output.py index 3c184a16a6..49d1ba7cb0 100644 --- a/verifiers/v1/cli/output.py +++ b/verifiers/v1/cli/output.py @@ -20,7 +20,7 @@ from verifiers.v1.configs.cli.eval import EvalConfig from verifiers.v1.episode import Episode, WireEpisode -from verifiers.v1.trace import Trace +from verifiers.v1.trace import EXCLUDE_FIELDS, Trace from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.install import env_name @@ -74,9 +74,12 @@ def save_config(config: BaseModel, results_dir: Path) -> None: def write_episode(results_dir: Path, episode: Episode) -> None: - """Serialize and append one rollout episode in the worker thread.""" + """Serialize and append one rollout episode in the worker thread. Raw per-node tensors stay + out of the record — they are the trainer's, and numpy bytes do not round-trip json.""" # Preserve fields declared by typed Trace subclasses nested in the episode. - data = type_adapter(type(episode)).dump_json(episode, exclude_none=True) + data = type_adapter(type(episode)).dump_json( + episode, exclude_none=True, exclude={"traces": {"__all__": EXCLUDE_FIELDS}} + ) with (results_dir / TRACES_FILE).open("ab") as f: f.write(data + b"\n")