diff --git a/environments/proposer_solver_v1/proposer_solver_v1/taskset.py b/environments/proposer_solver_v1/proposer_solver_v1/taskset.py index 06d49324e9..b023f8dbb1 100644 --- a/environments/proposer_solver_v1/proposer_solver_v1/taskset.py +++ b/environments/proposer_solver_v1/proposer_solver_v1/taskset.py @@ -152,7 +152,9 @@ def _solve_rate(traces: list[vf.Trace]) -> float: solves = [t for t in traces if t.agent_name == "solver"] if not solves: return 0.0 - return sum(t.rewards.get("correct", 0.0) for t in solves) / len(solves) + return sum( + r.score if (r := t.rewards.get("correct")) else 0.0 for t in solves + ) / len(solves) async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: """The proposer is judged by what its problem DOES to the solvers: diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 5b13941508..0c77546ae4 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -197,7 +197,7 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path): ) assert trace.ok, trace.errors assert trace.stop_condition == "user_closed" - assert trace.rewards["resumed"] == 1.0 + assert trace.rewards["resumed"].score == 1.0 assert trace.tools # ACP-native tools, or Pi's MCP adapter meta-tool segments = trace.info["acp_segments"] assert len(segments) == 2 @@ -320,7 +320,7 @@ async def test_rubric_judge(run_v1, tmp_path): max_turns=2, ) assert trace.ok - assert trace.rewards["rubric"] > 0 # the judge's verdict landed in the reward + assert trace.rewards["rubric"].score > 0 # the judge's verdict landed in the reward assert trace.metrics["rubric/always_yes"] == 1.0 assert trace.info["judge"] # the call was recorded onto the trace @@ -428,9 +428,11 @@ async def test_env_id_agentic_judge(run_v1, tmp_path): (judge,) = [t for t in traces if t.agent_name == "judge"] assert solver.ok and judge.ok assert judge.trainable is False - assert solver.rewards["echoed"] == 0.5 # the task's own reward, rescaled + # The task's own reward keeps its raw score; the rescale lands on the weight. + assert solver.rewards["echoed"].score == 1.0 + assert solver.rewards["echoed"].weight == 0.5 assert isinstance(judge.info.get("verdict"), dict) # scraped off the box - assert 0.0 <= solver.rewards["judge"] <= 1.0 + assert 0.0 <= solver.rewards["judge"].score <= 1.0 @pytest.mark.e2e @@ -491,7 +493,7 @@ async def test_env_id_user_sim_with_tools(run_v1, tmp_path): assert assistant.ok and user.ok assert assistant.task.data.prompt is None # the scenario stayed off the wire assert user.num_turns >= 1 # the modeled user actually drove the exchange - assert assistant.rewards["echoed"] == 1.0 # the tool ran, mid-conversation + assert assistant.rewards["echoed"].score == 1.0 # the tool ran, mid-conversation # The tool was advertised to the masked chat exactly as to any run. assert assistant.tools is not None assert any(tool.name == "echo_back" for tool in assistant.tools) @@ -513,7 +515,7 @@ async def test_kuhn_poker_self_play(run_v1, tmp_path): rollout_timeout=300, ) assert sorted(t.agent_name for t in traces) == ["player0", "player1"] - payoffs = {t.agent_name: t.rewards["payoff"] for t in traces} + payoffs = {t.agent_name: t.rewards["payoff"].score for t in traces} assert payoffs["player0"] + payoffs["player1"] == 0 # zero-sum assert abs(payoffs["player0"]) in (1.0, 2.0) for trace in traces: diff --git a/tests/v1/test_judges.py b/tests/v1/test_judges.py index cc0860b027..cf5e2f0b6f 100644 --- a/tests/v1/test_judges.py +++ b/tests/v1/test_judges.py @@ -453,7 +453,7 @@ async def gibberish_judge( # model failure: empty reply -> judge skipped, reward 0.0, NO error trace = make_trace(reply="") await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None) - assert trace.rewards["reference"] == 0.0 + assert trace.rewards["reference"].score == 0.0 # judge failure: unparseable verdict -> the rollout errors, no reward recorded trace = make_trace() with pytest.raises(vf.TaskError, match="no yes/no verdict"): @@ -623,11 +623,13 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model): taskset = JudgedTaskset(cfg) trace = make_trace() await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None) - assert trace.rewards["own"] == 0.25 # decorated rewards still run + assert trace.rewards["own"].score == 0.25 # decorated rewards still run + assert trace.rewards["reference"] == vf.Reward( + score=1.0, weight=0.5 + ) # raw score + weight, under the id-derived name assert ( - trace.rewards["reference"] == 0.5 - ) # 1.0 * weight 0.5, under the id-derived name - assert trace.rewards["quality"] == 0.75 # the rubric's aggregate, under its `name` + trace.rewards["quality"].score == 0.75 + ) # the rubric's aggregate, under its `name` assert ( len(trace.info["judge"]) == 2 ) # every judge call recorded (rubric = one call) @@ -636,4 +638,4 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model): async def test_task_without_judges_scores_as_before(): trace = make_trace() await JudgedTask(trace.task.data).score(trace, runtime=None) - assert trace.rewards == {"own": 0.25} + assert trace.rewards == {"own": vf.Reward(score=0.25)} diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 2b07436612..55ab5998b7 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -107,6 +107,7 @@ EvalRunInfo, GenerationSpan, ModelCall, + Reward, RunInfo, TimeSpan, TimeSplit, @@ -171,6 +172,7 @@ "Trace", "TraceTask", "WireTrace", + "Reward", "Episode", "WireEpisode", "TRACE_VERSION", diff --git a/verifiers/v1/cli/dashboard/eval.py b/verifiers/v1/cli/dashboard/eval.py index fcb1a42265..67f2ab7f02 100644 --- a/verifiers/v1/cli/dashboard/eval.py +++ b/verifiers/v1/cli/dashboard/eval.py @@ -341,13 +341,19 @@ def _score_segments(traces: list[Trace], source: str) -> str | None: return None segments = [] for name in names: - mean = format_mean( - traces, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0) - ) + mean = format_mean(traces, lambda t, n=name, s=source: _score(t, s, n)) segments.append(f"{name} {mean}") return " · ".join(segments) +def _score(trace: Trace, source: str, name: str) -> float: + """Rewards carry raw score + weight; the breakdown shows the raw score.""" + if source == "rewards": + reward = trace.rewards.get(name) + return reward.score if reward is not None else 0.0 + return trace.metrics.get(name, 0.0) + + def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None: """Score rows read the policy view (`scored` — trainable traces); with several roles in play they split per role, each role averaging over its OWN traces (no diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py index e3e5fc3473..3ad1a66e70 100644 --- a/verifiers/v1/envs/agentic_judge/env.py +++ b/verifiers/v1/envs/agentic_judge/env.py @@ -340,8 +340,8 @@ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None: for criterion in criteria: solution.record_metric(f"judge/{criterion.name}", scores[criterion.name]) if self.config.score.task_weight != 1.0: - for name in solution.rewards: - solution.rewards[name] *= self.config.score.task_weight + for reward in solution.rewards.values(): + reward.weight *= self.config.score.task_weight total = sum(criterion.weight for criterion in criteria) reward = sum(c.weight * scores[c.name] for c in criteria) / total solution.record_reward("judge", reward, weight=self.config.score.judge_weight) diff --git a/verifiers/v1/legacy.py b/verifiers/v1/legacy.py index 9fe73f644a..32f5866eaf 100644 --- a/verifiers/v1/legacy.py +++ b/verifiers/v1/legacy.py @@ -37,6 +37,7 @@ Error, GenerationSpan, ModelCall, + Reward, TimeSpan, TimeSplit, Timing, @@ -270,7 +271,7 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace: data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")), ), tools=_to_v1_tools(out.get("tool_defs")), - rewards={"reward": float(out.get("reward") or 0.0)}, + rewards={"reward": Reward(score=float(out.get("reward") or 0.0))}, metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()}, info=dict(out.get("info") or {}), is_completed=bool(out.get("is_completed", True)), diff --git a/verifiers/v1/push.py b/verifiers/v1/push.py index 49683b0447..f16fde4745 100644 --- a/verifiers/v1/push.py +++ b/verifiers/v1/push.py @@ -90,9 +90,10 @@ def dump(messages): else None, "info": dict(trace.info) or None, } - # Flatten sub-rewards to top-level keys the way v0 does; env metrics stay nested. - for name, value in trace.rewards.items(): - sample.setdefault(name, value) + # Flatten sub-rewards to top-level keys the way v0 does (raw scores, as v0's + # per-function outputs were); env metrics stay nested. + for name, reward in trace.rewards.items(): + sample.setdefault(name, reward.score) return sample @@ -127,7 +128,8 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any] sums: dict[str, float] = {} counts: dict[str, int] = {} for trace in scored: - for name, value in {**trace.rewards, **trace.metrics}.items(): + scores = {name: reward.score for name, reward in trace.rewards.items()} + for name, value in {**scores, **trace.metrics}.items(): sums[name] = sums.get(name, 0.0) + value counts[name] = counts.get(name, 0) + 1 n = len(scored) diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index c5c13d57af..ae90009e99 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -266,7 +266,7 @@ def num_input_tokens(self) -> int: """Raw tensor fields kept on the msgpack wire but excluded from JSON records.""" -TRACE_VERSION = 3 +TRACE_VERSION = 4 """Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on breaking shape changes; optional-with-default fields are additive and don't bump it.""" @@ -343,6 +343,21 @@ class TraceTask(StrictBaseModel, Generic[DataT]): """The (immutable) row being solved.""" +class Reward(StrictBaseModel): + """One named reward as recorded on the trace: the raw score next to its weight, + so records keep both readable and the weighted sum stays a derived view.""" + + score: float + """The raw value the reward function returned, unweighted.""" + weight: float = 1.0 + """The multiplier `score` carries in the trace-level `reward` sum.""" + + @property + def value(self) -> float: + """This reward's weighted contribution to the trace-level `reward`.""" + return self.score * self.weight + + class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]): id: str = Field(default_factory=lambda: uuid.uuid4().hex) """Unique id for this rollout, auto-generated per trace.""" @@ -369,8 +384,9 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]): """Every provider exchange behind the sampled turns, in order: raw wire request/response plus per-call timing and errors, linked into `nodes` via `ModelCall.node`.""" - rewards: dict[str, float] = Field(default_factory=dict) - """Weighted contributions from task rewards, judges, and the env's `score()`.""" + rewards: dict[str, Reward] = Field(default_factory=dict) + """Named rewards from tasks, judges, and the env's `score()` — each keeps its + raw `score` and `weight`; the trace-level `reward` is their weighted sum.""" metrics: dict[str, float] = Field(default_factory=dict) """Unweighted metrics from tasks, harnesses, and judges.""" info: dict[str, Any] = Field(default_factory=dict) @@ -400,7 +416,7 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]): @property def reward(self) -> float: - return sum(self.rewards.values()) + return sum(r.value for r in self.rewards.values()) @property def error(self) -> Error | None: @@ -559,12 +575,12 @@ def record_judge(self, response: JudgeResponse) -> None: self.extra_usage.append(response.usage) def record_reward(self, name: str, value: float, weight: float = 1.0) -> None: - contribution = float(value) * float(weight) + reward = Reward(score=float(value), weight=float(weight)) if name in self.rewards: logger.warning( - "reward %r overridden: %s -> %s", name, self.rewards[name], contribution + "reward %r overridden: %s -> %s", name, self.rewards[name], reward ) - self.rewards[name] = contribution + self.rewards[name] = reward def stamp(self, run: RunInfo | None = None, **info: Any) -> None: """Stamp identity only the consumer knows (the eval CLI / a trainer) onto the