feat(v1): run metadata on episode

verifiers/v1/__init__.py

@@ -30,7 +30,18 @@
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
-from verifiers.v1.episode import Episode, WireEpisode
+from verifiers.v1.episode import (
+    EnvInfo,
+    Episode,
+    EvalMetadata,
+    EvalRunInfo,
+    Metadata,
+    PolicySpan,
+    RunInfo,
+    TrainMetadata,
+    TrainRunInfo,
+    WireEpisode,
+)
 from verifiers.v1.errors import (
     EnvError,
     HarnessError,
@@ -75,16 +86,13 @@
     AgentSpan,
     Branch,
     Error,
-    EvalRunInfo,
     ModelCall,
     Reward,
-    RunInfo,
     TimeSpan,
     TimeSplit,
     Timing,
     Trace,
     TraceTask,
-    TrainRunInfo,
     VersionInfo,
     WireTrace,
 )
@@ -194,11 +202,16 @@
     "WireTrace",
     "Reward",
     "Episode",
+    "EnvInfo",
     "WireEpisode",
     "TRACE_VERSION",
     "AgentInfo",
     "RunInfo",
     "EvalRunInfo",
+    "EvalMetadata",
+    "Metadata",
+    "TrainMetadata",
+    "PolicySpan",
     "ModelCall",
     "TrainRunInfo",
     "VersionInfo",

verifiers/v1/cli/eval/runner.py

@@ -16,9 +16,8 @@
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
-from verifiers.v1.episode import Episode
+from verifiers.v1.episode import Episode, EvalRunInfo
 from verifiers.v1.taskset import SEED
-from verifiers.v1.trace import EvalRunInfo
 
 logger = logging.getLogger(__name__)
 

verifiers/v1/cli/output.py

@@ -20,7 +20,7 @@
 
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
-from verifiers.v1.trace import Trace
+from verifiers.v1.trace import EXCLUDE_FIELDS, Trace
 from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.install import env_name
 
@@ -74,9 +74,12 @@ def save_config(config: BaseModel, results_dir: Path) -> None:
 
 
 def write_episode(results_dir: Path, episode: Episode) -> None:
-    """Serialize and append one rollout episode in the worker thread."""
+    """Serialize and append one rollout episode in the worker thread. Raw per-node tensors stay
+    out of the record — they are the trainer's, and numpy bytes do not round-trip json."""
     # Preserve fields declared by typed Trace subclasses nested in the episode.
-    data = type_adapter(type(episode)).dump_json(episode, exclude_none=True)
+    data = type_adapter(type(episode)).dump_json(
+        episode, exclude_none=True, exclude={"traces": {"__all__": EXCLUDE_FIELDS}}
+    )
     with (results_dir / TRACES_FILE).open("ab") as f:
         f.write(data + b"\n")
 

verifiers/v1/episode.py

@@ -1,14 +1,14 @@
 """The episode — one run's traces plus their shared standing, whole."""
 
 import uuid
-from typing import Any, Generic
+from typing import Annotated, Any, Generic, Literal
 
 from pydantic import BaseModel, Field
 
 from verifiers.v1.configs.agent import WireAgentConfig
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import DataT, WireTaskData
-from verifiers.v1.trace import AgentConfigT, Error, RunInfo, Trace
+from verifiers.v1.trace import AgentConfigT, Error, Trace
 from verifiers.v1.types import Usage
 
 
@@ -18,6 +18,79 @@ class EnvInfo(BaseModel):
     id: str = ""
     """`EnvConfig.env_id`, e.g. `agentic-judge+gsm8k-v1`."""
 
+    name: str | None = None
+    """What the caller knows this env by, when that differs from `id`."""
+
+
+class EvalRunInfo(BaseModel):
+    """A standalone eval: a model measured against nothing that is training."""
+
+    type: Literal["eval"] = "eval"
+    id: str
+
+
+class PolicySpan(BaseModel):
+    """Live policy versions with a derived, non-serialized drift in updates."""
+
+    start: int = 0
+    end: int = 0
+
+    @property
+    def drift(self) -> int:
+        return max(0, self.end - self.start)
+
+
+class Metadata(BaseModel):
+    """What one episode is to the training run it belongs to."""
+
+    policy: PolicySpan | None = None
+    """`None` when it was not generated from the live policy, as with a frozen sampler."""
+
+
+class TrainMetadata(Metadata):
+    """An episode a training run trains on."""
+
+    type: Literal["train"] = "train"
+    step: int | None = None
+    """The batch window it landed in, which is not known until it does."""
+
+    @property
+    def off_policy_steps(self) -> int | None:
+        """Versions behind the policy in training, queue time included."""
+        if self.policy is None or self.step is None:
+            return None
+        return max(0, (self.step - 1) - self.policy.start)
+
+
+class EvalMetadata(Metadata):
+    """An episode a training run measures itself with."""
+
+    type: Literal["eval"] = "eval"
+    step: int
+    """The step whose eval produced it, known when it is dispatched."""
+
+    @property
+    def off_policy_steps(self) -> int | None:
+        """Versions the policy moved under it. Nothing trains on an eval, so it can only be
+        off-policy by drifting — and its `step` is fixed at dispatch, so it cannot say that."""
+        return self.policy.drift if self.policy else None
+
+
+EpisodeMetadata = Annotated[TrainMetadata | EvalMetadata, Field(discriminator="type")]
+
+
+class TrainRunInfo(BaseModel):
+    """A training run: one id over the episodes it trains on and the ones it evaluates itself with,
+    which `metadata` tells apart."""
+
+    type: Literal["train"] = "train"
+    id: str
+    metadata: EpisodeMetadata = Field(default_factory=TrainMetadata)
+
+
+RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")]
+"""The run an episode belongs to, discriminated on `type`."""
+
 
 class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     """The artifact Env.run produces. Contains multiple agents' traces."""

verifiers/v1/trace.py

@@ -4,7 +4,7 @@
 import traceback
 import uuid
 from collections.abc import Callable, Mapping
-from typing import TYPE_CHECKING, Annotated, Any, Generic, Literal
+from typing import TYPE_CHECKING, Any, Generic
 
 import numpy as np
 from pydantic import BaseModel, Field, PrivateAttr
@@ -132,24 +132,6 @@ def value(self) -> float:
         return self.score * self.weight
 
 
-class EvalRunInfo(BaseModel):
-    type: Literal["eval"] = "eval"
-
-    id: str
-    step: int | None = None
-
-
-class TrainRunInfo(BaseModel):
-    type: Literal["train"] = "train"
-
-    id: str
-    step: int | None = None
-
-
-RunInfo = Annotated[EvalRunInfo | TrainRunInfo, Field(discriminator="type")]
-"""The run a trace belongs to, discriminated on `type`."""
-
-
 class ModelCall(BaseModel):
     """A model call, automatically recorded at intercept time."""