Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions docs/guides/grpo.md
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,22 @@ grpo:
```
Note that this method only has an effect when training involves more than one reward function.

#### Per-reward weights

The aggregation weights \\( w_n \\) in \\( A^{(i,j)} = \sum_{n=1}^{N} w_n A_n^{(i,j)} \\)
are configurable via `reward_weights`, an optional list with one entry per reward
component, ordered to match `reward1, reward2, ...`. When omitted, all weights default
to `1.0` (equal weighting). For example, to down-weight the second and third rewards:
```
grpo:
adv_estimator:
name: "gdpo"
normalize_rewards: true
use_leave_one_out_baseline: false
reward_weights: [1.0, 0.5, 0.25]
Comment thread
terrykong marked this conversation as resolved.
Resolved
```
The number of weights must equal the number of reward components, or a `ValueError` is raised.

## LoRA Configuration

GRPO supports LoRA on both the DTensor and Megatron backends. To enable LoRA on the default DTensor backend:
Expand Down
34 changes: 34 additions & 0 deletions examples/nemo_gym/gdpo_multireward.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# GDPO over a multi-reward NeMo Gym environment.
#
# Inherits the workplace-assistant NeMo Gym recipe and switches only the advantage
# estimator to GDPO. GDPO requires the Gym environment to return multiple reward
# components: its verify endpoint must include `reward_components` (a mapping of
# component-name -> score) in addition to the scalar `reward`. The Gym->NeMo-RL bridge
# (nemo_rl/environments/nemo_gym.py::extract_reward_components, surfaced in
# run_async_nemo_gym_rollout) then exposes them as reward1, reward2, ... for the GDPO
# advantage estimator.
#
# NOTE: workplace_assistant is single-reward today; point `env.nemo_gym` at a
# multi-reward environment (one whose verifier emits `reward_components`) before
# training, or GDPO will raise "GDPO requires multiple reward components". A ready
# reference env is `example_tool_call_multireward` in NVIDIA-NeMo/Gym (scored on
# correctness / schema_valid / format). See
# skills/nemo-rl-gdpo-experiment/references/gym-multireward-scope.md.
Comment thread
terrykong marked this conversation as resolved.
Resolved
defaults: grpo_workplace_assistant_nemotron_nano_v2_9b.yaml

grpo:
adv_estimator:
name: "gdpo"
normalize_rewards: true
use_leave_one_out_baseline: false
# Optional per-component weights, ordered to match reward1, reward2, ... .
# Omit for equal weighting; uncomment and size to your env's components to tune.
# reward_weights: [1.0, 0.5, 0.25]

checkpointing:
checkpoint_dir: "results/gdpo_nemo_gym"

logger:
wandb:
project: "gdpo-dev"
name: "gdpo-nemo-gym-multireward"
19 changes: 17 additions & 2 deletions nemo_rl/algorithms/advantage_estimator.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

This module provides different advantage estimation strategies:
- GRPOAdvantageEstimator: Standard GRPO advantage with leave-one-out baseline
- GDPOAdvantageEstimator: Multi-reward GDPO (per-component baselines, sum then normalize)
- GDPOAdvantageEstimator: Multi-reward GDPO (per-component baselines, optional per-reward weights, then normalize)
- ReinforcePlusPlusAdvantageEstimator: Reinforce++ with optional baseline subtraction (minus_baseline) and KL penalty in reward
- RawRewardAdvantageEstimator: Raw reward as advantage with optional batch normalization (no baseline, no value model)
- GeneralizedAdvantageEstimator: Generalized Advantage Estimation (GAE) with temporal bootstrapping
Expand Down Expand Up @@ -91,6 +91,9 @@ class GDPOAdvantageEstimator:
def __init__(self, estimator_config: dict, loss_config: ClippedPGLossConfig):
self.use_leave_one_out_baseline = estimator_config["use_leave_one_out_baseline"]
self.normalize_rewards = estimator_config["normalize_rewards"]
# Optional per-reward weights w_n for the aggregation A = sum_n w_n * A_n
# (paper: https://arxiv.org/abs/2601.05242). None => equal weights (all 1.0).
self.reward_weights = estimator_config.get("reward_weights")

def compute_advantage(
self,
Expand Down Expand Up @@ -119,6 +122,16 @@ def compute_advantage(
f"This batch has {len(reward_component_keys)} component(s): {reward_component_keys}. "
"Switch to GRPO by setting grpo.adv_estimator.name to 'grpo' in your config."
)
# Resolve per-reward weights (ordered to match reward1, reward2, ...).
weights = self.reward_weights
if weights is None:
weights = [1.0] * len(reward_component_keys)
elif len(weights) != len(reward_component_keys):
raise ValueError(
f"reward_weights has {len(weights)} entries but this batch has "
f"{len(reward_component_keys)} reward components ({reward_component_keys}). "
"Provide exactly one weight per component, ordered to match reward1, reward2, ..."
)
valid = torch.ones_like(repeated_batch[reward_component_keys[0]])
leave_one_out = self.use_leave_one_out_baseline
assert prompt_ids.shape[0] == valid.shape[0], (
Expand All @@ -144,7 +157,9 @@ def compute_advantage(

advantage_parts.append(adv_k)

advantages = sum(advantage_parts)
advantages = sum(
weight * adv_k for weight, adv_k in zip(weights, advantage_parts)
)
# Normalize combined advantage to zero mean and unit std
adv_std = advantages.std()
if adv_std > 0:
Expand Down
4 changes: 4 additions & 0 deletions nemo_rl/algorithms/grpo.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,10 @@ class AdvEstimatorConfig(TypedDict):
# GRPO specific
normalize_rewards: NotRequired[bool]
use_leave_one_out_baseline: NotRequired[bool]
# GDPO specific: optional per-component weights w_n for the aggregation
# A = sum_n w_n * A_n, ordered to match reward1, reward2, ... . Defaults to
# equal weights (all 1.0) when omitted.
reward_weights: NotRequired[list[float] | None]
# Reinforce++ specific
minus_baseline: NotRequired[bool]

Expand Down
45 changes: 45 additions & 0 deletions nemo_rl/environments/nemo_gym.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
import os
import subprocess
from pathlib import Path
Expand Down Expand Up @@ -469,6 +470,50 @@ def global_post_process_and_metrics(self, batch):
raise NotImplementedError


def extract_reward_components(nemo_gym_result: dict) -> Dict[str, float] | None:
"""Return per-component rewards from a NeMo Gym verify result, or None.

Single-reward NeMo Gym environments return only a scalar ``reward``. Multi-reward
environments additionally return ``reward_components``: a mapping of
component-name -> score. These are consumed by GDPO as reward1, reward2, ...
(see ``nemo_rl.algorithms.advantage_estimator.GDPOAdvantageEstimator``).

Returns ``None`` when the environment is single-reward (no ``reward_components``),
so callers fall back to the scalar ``reward`` path unchanged.
"""
components = nemo_gym_result.get("reward_components")
if not components:
return None
return {str(name): float(score) for name, score in components.items()}


def validate_reward_components_match_scalar(nemo_gym_results: List[dict]) -> None:
"""Assert each multi-reward result sets ``reward == sum(reward_components)``.

A multi-reward verifier must set the scalar ``reward`` to the sum of its
``reward_components`` so single-reward (GRPO) consumers and GDPO read the same
aggregate. We keep the verifier's scalar ``reward`` as ``total_reward`` rather than
silently overwriting it with the component sum, so a verifier that violates this
contract must be surfaced here instead of masked.

Raises ``ValueError`` on the first violating result. A no-op for single-reward
results (those without ``reward_components``).
"""
for idx, result in enumerate(nemo_gym_results):
components = extract_reward_components(result)
if components is None:
continue
scalar_reward = float(result["reward"])
component_sum = sum(components.values())
if not math.isclose(scalar_reward, component_sum, rel_tol=1e-5, abs_tol=1e-6):
raise ValueError(
f"NeMo Gym verify result {idx} has reward={scalar_reward} but its "
f"reward_components sum to {component_sum} ({components}). A multi-reward "
"verifier must set reward = sum(reward_components.values()) so single-reward "
"(GRPO) consumers and GDPO read the same aggregate."
)


########################################
# Global config utils
########################################
Expand Down
34 changes: 34 additions & 0 deletions nemo_rl/experience/rollouts.py
Original file line number Diff line number Diff line change
Expand Up @@ -2052,6 +2052,40 @@ def run_async_nemo_gym_rollout(
if _get_reward_penalty_config_value(resolved_reward_penalty_config, flag):
rollout_metrics[metric_name] = penalty_counts[key] / len(results)

# Expose per-component rewards (reward1, reward2, ...) for multi-reward NeMo Gym
# environments so GDPO can compute per-component advantages; single-reward envs are
# unaffected. Mirrors the native rollout path's reward-component handling above.
from nemo_rl.environments.nemo_gym import (
extract_reward_components,
validate_reward_components_match_scalar,
)

component_dicts = [extract_reward_components(r["full_result"]) for r in results]
if any(c is not None for c in component_dicts):
# Canonical (alphabetical) component ordering, not just deduplication: the
# reward{n} index -> component-name mapping must be stable across every sample
# and every run. GDPO's per-component baseline (calculate_baseline_and_std_per_prompt)
# assumes reward1 denotes the same signal for all responses to a prompt, and
# grpo.adv_estimator.reward_weights is documented as "ordered to match reward1,
# reward2, ...". A first-seen order (e.g. dict.fromkeys) would be batch-dependent
# and could shift that mapping between runs; sorting keeps it deterministic.
ordered_names = sorted(
Comment thread
anjalibshah marked this conversation as resolved.
Resolved
{name for c in component_dicts if c is not None for name in c}
)
for component_idx, name in enumerate(ordered_names, start=1):
final_batch[f"reward{component_idx}"] = torch.tensor(
Comment thread
terrykong marked this conversation as resolved.
Resolved
[
c[name] if c is not None and name in c else 0.0
for c in component_dicts
]
)
# Leave total_reward as the verifier's scalar `reward` (set above); do not
# silently overwrite it. When a verifier emits reward_components, the contract is
# reward == sum(components), so overwriting would be a no-op in the correct case
# and would only mask a misconfigured verifier when it isn't. Validate that
# contract instead and fail fast on a real mismatch.
validate_reward_components_match_scalar([r["full_result"] for r in results])
Comment thread
terrykong marked this conversation as resolved.
Resolved

return AsyncNemoGymRolloutResult(
input_ids=input_ids,
final_batch=final_batch,
Expand Down
30 changes: 30 additions & 0 deletions tests/unit/algorithms/test_grpo.py
Original file line number Diff line number Diff line change
Expand Up @@ -2861,6 +2861,36 @@ def test_gdpo_advantage_estimator_single_reward():
estimator.compute_advantage(prompt_ids, None, mask, repeated_batch)


def test_gdpo_advantage_estimator_reward_weights():
"""GDPO per-reward weights: uniform weights match the default; non-uniform differ; wrong length raises."""
loss_config = ClippedPGLossConfig()
prompt_ids = torch.tensor([[0], [0], [0]])
mask = torch.ones(3, 2)
repeated_batch = {
"reward1": torch.tensor([1.0, 0.0, 1.0]),
"reward2": torch.tensor([1.0, 1.0, 0.0]),
}

def run(weights):
config = {"use_leave_one_out_baseline": False, "normalize_rewards": True}
if weights is not None:
config["reward_weights"] = weights
estimator = GDPOAdvantageEstimator(config, loss_config)
return estimator.compute_advantage(prompt_ids, None, mask, dict(repeated_batch))

default = run(None)

# Any positive uniform scaling is invariant after the final per-batch normalization.
assert torch.allclose(default, run([2.0, 2.0]), atol=1e-5)

# Non-uniform weights change the advantages.
assert not torch.allclose(default, run([1.0, 0.25]), atol=1e-3)

# Wrong number of weights -> ValueError.
with pytest.raises(ValueError):
run([1.0])


# ============================================================================
# Tests for ReinforcePlusPlusAdvantageEstimator class
# ============================================================================
Expand Down
53 changes: 53 additions & 0 deletions tests/unit/environments/test_nemo_gym.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
from nemo_rl.environments.nemo_gym import (
NemoGym,
NemoGymConfig,
extract_reward_components,
setup_nemo_gym_config,
validate_reward_components_match_scalar,
)
from nemo_rl.models.generation.vllm import VllmGeneration

Expand All @@ -43,6 +45,57 @@
)


def test_extract_reward_components():
"""The GDPO multi-reward bridge helper: None for single-reward, normalized dict otherwise."""
# Single-reward result (no reward_components) -> None, so callers use scalar reward.
assert extract_reward_components({"reward": 1.0}) is None
assert extract_reward_components({"reward": 1.0, "reward_components": {}}) is None

# Multi-reward result -> name->float dict (values coerced to float, keys to str).
components = extract_reward_components(
{
"reward": 2.0,
"reward_components": {"correctness": 1, "format": 0.5},
}
)
assert components == {"correctness": 1.0, "format": 0.5}
assert all(isinstance(v, float) for v in components.values())


def test_validate_reward_components_match_scalar():
"""Multi-reward verifiers must set reward == sum(reward_components); mismatch raises."""
# Contract satisfied: reward equals the component sum -> no error.
validate_reward_components_match_scalar(
[{"reward": 1.5, "reward_components": {"correctness": 1.0, "format": 0.5}}]
)
# Float tolerance: tiny rounding differences are accepted.
validate_reward_components_match_scalar(
[
{
"reward": 1.5000001,
"reward_components": {"correctness": 1.0, "format": 0.5},
}
]
)
# Single-reward results (no reward_components) are skipped entirely.
validate_reward_components_match_scalar([{"reward": 2.0}])

# Mismatch (scalar reward != component sum) -> ValueError naming the offending index.
with pytest.raises(ValueError, match="result 1"):
validate_reward_components_match_scalar(
[
{
"reward": 1.5,
"reward_components": {"correctness": 1.0, "format": 0.5},
},
{
"reward": 2.0,
"reward_components": {"correctness": 1.0, "format": 0.5},
},
]
)


@pytest.mark.nemo_gym
def test_nemo_gym_stub_module():
from nemo_gym import config_types
Expand Down
Loading