feat(v1): run metadata on episode

verifiers/v1/__init__.py

@@ -30,7 +30,7 @@
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
-from verifiers.v1.episode import Episode, WireEpisode
+from verifiers.v1.episode import Episode, GroupInfo, WireEpisode
 from verifiers.v1.errors import (
     EnvError,
     HarnessError,
@@ -194,6 +194,7 @@
     "WireTrace",
     "Reward",
     "Episode",
+    "GroupInfo",
     "WireEpisode",
     "TRACE_VERSION",
     "AgentInfo",

verifiers/v1/env.py

@@ -20,7 +20,7 @@
     _declared_agent_configs,
     default_agent_harness,
 )
-from verifiers.v1.episode import EnvInfo, Episode
+from verifiers.v1.episode import EnvInfo, Episode, GroupInfo
 from verifiers.v1.errors import EnvError, boundary
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.interception import (
@@ -56,6 +56,8 @@ class RunSlot:
     traces: list[Trace] = field(default_factory=list)
     episode: Episode | None = None
     done: bool = False
+    group: GroupInfo | None = None
+    """The cohort this slot was planned in, stamped onto its episode when it lands."""
 
     @classmethod
     def finished(cls, episode: Episode) -> "RunSlot":
@@ -64,6 +66,7 @@ def finished(cls, episode: Episode) -> "RunSlot":
             traces=list(episode.traces),
             episode=episode,
             done=True,
+            group=episode.group,
         )
 
 
@@ -294,10 +297,13 @@ async def run_episode(
         return episode
 
     def slots(self, task: Task, n: int = 1) -> list[RunSlot]:
-        """Plan `n` independent episodes of `task` (`-r n`): nothing couples them."""
+        """Plan `n` independent episodes of `task` (`-r n`). They run independently, but they
+        are one cohort — the attempts a consumer compares with each other — so they share a
+        `GroupInfo`, which lands on each episode."""
         if n < 1:
             raise ValueError("a task needs at least one rollout (n >= 1)")
-        return [RunSlot(task) for _ in range(n)]
+        group = GroupInfo(size=n)
+        return [RunSlot(task, group=group) for _ in range(n)]
 
     async def run_slot(
         self,
@@ -330,6 +336,7 @@ def discard(trace: Trace) -> None:
                 )
 
         episode = await run_episode_with_retry(attempt, self.config.retries)
+        episode.group = slot.group
         slot.traces = list(episode.traces)
         slot.episode = episode
         slot.done = True

verifiers/v1/episode.py

@@ -18,6 +18,24 @@ class EnvInfo(BaseModel):
     id: str = ""
     """`EnvConfig.env_id`, e.g. `agentic-judge+gsm8k-v1`."""
 
+    name: str = ""
+    """What the caller knows this env by, when that differs from `id` — the key it was
+    configured under. A run over several envs keys its metrics by this; a run over one
+    can leave it empty."""
+
+
+class GroupInfo(BaseModel):
+    """The cohort an episode belongs to: the episodes planned together from one task, which a
+    consumer compares against each other (pass@k over the group, a GRPO baseline within it).
+
+    Its `id` is per cohort, not per task: the same task planned again later is a new group, so
+    two rounds of it never merge into one comparison."""
+
+    id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+
+    size: int = 1
+    """How many episodes were planned in it — the `k` of `-r k`, before any of them fail."""
+
 
 class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     """The artifact Env.run produces. Contains multiple agents' traces."""
@@ -26,6 +44,9 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     env: EnvInfo = Field(default_factory=EnvInfo)
     """The env that produced this episode."""
+    group: GroupInfo | None = None
+    """The cohort it was planned in, when the producer planned one — several episodes of the
+    same task, meant to be compared with each other."""
     run: RunInfo | None = None
     """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than
     on each trace because the episode is what a consumer dispatches, and an episode that produced