diff --git a/deps/verifiers b/deps/verifiers index 576506d66f..dd2950909b 160000 --- a/deps/verifiers +++ b/deps/verifiers @@ -1 +1 @@ -Subproject commit 576506d66f37a2d6d996a85b8eb2ef53bbfddd9f +Subproject commit dd2950909bc742a60505a355cc779cf2f39a03b0 diff --git a/src/prime_rl/orchestrator/metrics.py b/src/prime_rl/orchestrator/metrics.py index c877b7edf8..4835f10a81 100644 --- a/src/prime_rl/orchestrator/metrics.py +++ b/src/prime_rl/orchestrator/metrics.py @@ -134,8 +134,11 @@ def stats(self) -> dict[str, Stat]: class CustomMetrics(StatGroup): """Per-key ``Stat``s over a dynamic per-rollout dict attribute (env ``@metric``s or reward - components), each averaged over the rollouts that report the key. ``value`` extracts the - float from each entry (rewards are ``vf.Reward`` records; metrics are plain floats).""" + components), each over the rollouts that carry the key. Scoring seeds every expected key + with ``None`` before invoking it, so a ``None`` value means the signal never produced a + score and counts as 0.0 — the ``effective`` subset excludes errored rollouts and gives the + clean means. ``value`` extracts the float from each scored entry (rewards are ``vf.Reward`` + records; metrics are plain floats).""" def __init__(self, rollouts: list[Rollout], attr: str, value: Callable[[Any], float] = float) -> None: super().__init__(rollouts) @@ -146,7 +149,11 @@ def stats(self) -> dict[str, Stat]: names = sorted({name for r in self.rollouts for name in getattr(r, self.attr)}) return { name: Stat( - [self.value(getattr(r, self.attr)[name]) for r in self.rollouts if name in getattr(r, self.attr)] + [ + self.value(scores[name]) if scores[name] is not None else 0.0 + for r in self.rollouts + if name in (scores := getattr(r, self.attr)) + ] ) for name in names } diff --git a/tests/unit/orchestrator/test_metrics.py b/tests/unit/orchestrator/test_metrics.py index 3087721de5..579a1b79dc 100644 --- a/tests/unit/orchestrator/test_metrics.py +++ b/tests/unit/orchestrator/test_metrics.py @@ -144,13 +144,25 @@ def test_nested_metrics_and_rewards(): rollouts = [ mk(metrics={"acc": 1.0}, rewards={"correct": vf.Reward(score=1.0), "format": vf.Reward(score=0.0)}), mk(metrics={"acc": 3.0, "fmt": 5.0}, rewards={"correct": vf.Reward(score=0.0), "format": vf.Reward(score=1.0)}), + # scoring failed after seeding: unscored (None) entries count as 0.0 on `all` + mk(has_error=True, metrics={"acc": None}, rewards={"correct": None, "format": None}), ] - m = TrainRollouts(rollouts).metrics - assert m.metrics["acc"].mean() == 2.0 and m.rewards["correct"].mean() == 0.5 # nested group access + rc = TrainRollouts(rollouts) + m = rc.metrics + assert m.metrics["acc"].mean() == pytest.approx(4 / 3) and m.rewards["correct"].mean() == pytest.approx(1 / 3) out = m.to_wandb(prefix="train/agg", subset="all") - assert out["train/agg/all/metrics/acc/mean"] == 2.0 # averaged over reporters + assert out["train/agg/all/metrics/acc/mean"] == pytest.approx(4 / 3) assert out["train/agg/all/metrics/fmt/mean"] == 5.0 # single reporter - assert out["train/agg/all/rewards/format/mean"] == 0.5 + assert out["train/agg/all/rewards/format/mean"] == pytest.approx(1 / 3) + # effective drops the errored rollout, so its seeds don't dilute the effective means + eff = rc.effective.metrics.to_wandb(prefix="train/agg", subset="effective") + assert eff["train/agg/effective/metrics/acc/mean"] == 2.0 + assert eff["train/agg/effective/rewards/format/mean"] == 0.5 + # cross-env agg: another env's unscored trace carries different keys, so it can't dilute these + other = mk(env_name="other", has_error=True, rewards={"solved": None}) + agg = TrainRollouts(rollouts + [other]).metrics.to_wandb(prefix="train/agg", subset="all") + assert agg["train/agg/all/rewards/format/mean"] == pytest.approx(1 / 3) + assert agg["train/agg/all/rewards/solved/mean"] == 0.0 def test_nested_timing():