From cdf9b99650e70a4a1da6e4d212a2c1831f2688ba Mon Sep 17 00:00:00 2001 From: Anjali Shah Date: Tue, 2 Jun 2026 18:19:11 -0700 Subject: [PATCH 1/6] feat: configurable GDPO per-reward weights and multi-reward NeMo Gym bridge Add reward_weights to the GDPO advantage estimator so components aggregate as A = sum_n w_n * A_n (defaults to equal weights; wrong length raises). Extend the NeMo Gym bridge to surface per-component rewards (reward1..rewardN) from a verifier that returns reward_components, mirroring the native multi-reward rollout path, so GDPO can train on NeMo Gym environments. Includes unit tests and docs. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anjali Shah --- docs/guides/grpo.md | 16 ++++++++++++ examples/nemo_gym/gdpo_multireward.yaml | 32 +++++++++++++++++++++++ nemo_rl/algorithms/advantage_estimator.py | 19 ++++++++++++-- nemo_rl/algorithms/grpo.py | 4 +++ nemo_rl/environments/nemo_gym.py | 17 ++++++++++++ nemo_rl/experience/rollouts.py | 28 ++++++++++++++++++++ tests/unit/algorithms/test_grpo.py | 30 +++++++++++++++++++++ tests/unit/environments/test_nemo_gym.py | 18 +++++++++++++ 8 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 examples/nemo_gym/gdpo_multireward.yaml diff --git a/docs/guides/grpo.md b/docs/guides/grpo.md index 579a0af52a0..431966b8735 100755 --- a/docs/guides/grpo.md +++ b/docs/guides/grpo.md @@ -633,6 +633,22 @@ grpo: ``` Note that this method only has an effect when training involves more than one reward function. +#### Per-reward weights + +The aggregation weights \\( w_n \\) in \\( A^{(i,j)} = \sum_{n=1}^{N} w_n A_n^{(i,j)} \\) +are configurable via `reward_weights`, an optional list with one entry per reward +component, ordered to match `reward1, reward2, ...`. When omitted, all weights default +to `1.0` (equal weighting). For example, to down-weight the second and third rewards: +``` +grpo: + adv_estimator: + name: "gdpo" + normalize_rewards: true + use_leave_one_out_baseline: false + reward_weights: [1.0, 0.5, 0.25] +``` +The number of weights must equal the number of reward components, or a `ValueError` is raised. + ## LoRA Configuration GRPO supports LoRA on both the DTensor and Megatron backends. To enable LoRA on the default DTensor backend: diff --git a/examples/nemo_gym/gdpo_multireward.yaml b/examples/nemo_gym/gdpo_multireward.yaml new file mode 100644 index 00000000000..5911d1812f7 --- /dev/null +++ b/examples/nemo_gym/gdpo_multireward.yaml @@ -0,0 +1,32 @@ +# GDPO over a multi-reward NeMo Gym environment. +# +# Inherits the workplace-assistant NeMo Gym recipe and switches only the advantage +# estimator to GDPO. GDPO requires the Gym environment to return multiple reward +# components: its verify endpoint must include `reward_components` (a mapping of +# component-name -> score) in addition to the scalar `reward`. The Gym->NeMo-RL bridge +# (nemo_rl/environments/nemo_gym.py::extract_reward_components, surfaced in +# run_async_nemo_gym_rollout) then exposes them as reward1, reward2, ... for the GDPO +# advantage estimator. +# +# NOTE: workplace_assistant is single-reward today; point `env.nemo_gym` at a +# multi-reward environment (one whose verifier emits `reward_components`) before +# training, or GDPO will raise "GDPO requires multiple reward components". See +# skills/nemo-rl-gdpo-experiment/references/gym-multireward-scope.md. +defaults: grpo_workplace_assistant_nemotron_nano_v2_9b.yaml + +grpo: + adv_estimator: + name: "gdpo" + normalize_rewards: true + use_leave_one_out_baseline: false + # Optional per-component weights, ordered to match reward1, reward2, ... . + # Omit for equal weighting; uncomment and size to your env's components to tune. + # reward_weights: [1.0, 0.5, 0.25] + +checkpointing: + checkpoint_dir: "results/gdpo_nemo_gym" + +logger: + wandb: + project: "gdpo-dev" + name: "gdpo-nemo-gym-multireward" diff --git a/nemo_rl/algorithms/advantage_estimator.py b/nemo_rl/algorithms/advantage_estimator.py index 5a73902f329..0ab81a80f72 100644 --- a/nemo_rl/algorithms/advantage_estimator.py +++ b/nemo_rl/algorithms/advantage_estimator.py @@ -16,7 +16,7 @@ This module provides different advantage estimation strategies: - GRPOAdvantageEstimator: Standard GRPO advantage with leave-one-out baseline -- GDPOAdvantageEstimator: Multi-reward GDPO (per-component baselines, sum then normalize) +- GDPOAdvantageEstimator: Multi-reward GDPO (per-component baselines, optional per-reward weights, then normalize) - ReinforcePlusPlusAdvantageEstimator: Reinforce++ with optional baseline subtraction (minus_baseline) and KL penalty in reward - RawRewardAdvantageEstimator: Raw reward as advantage with optional batch normalization (no baseline, no value model) - GeneralizedAdvantageEstimator: Generalized Advantage Estimation (GAE) with temporal bootstrapping @@ -91,6 +91,9 @@ class GDPOAdvantageEstimator: def __init__(self, estimator_config: dict, loss_config: ClippedPGLossConfig): self.use_leave_one_out_baseline = estimator_config["use_leave_one_out_baseline"] self.normalize_rewards = estimator_config["normalize_rewards"] + # Optional per-reward weights w_n for the aggregation A = sum_n w_n * A_n + # (paper: https://arxiv.org/abs/2601.05242). None => equal weights (all 1.0). + self.reward_weights = estimator_config.get("reward_weights") def compute_advantage( self, @@ -119,6 +122,16 @@ def compute_advantage( f"This batch has {len(reward_component_keys)} component(s): {reward_component_keys}. " "Switch to GRPO by setting grpo.adv_estimator.name to 'grpo' in your config." ) + # Resolve per-reward weights (ordered to match reward1, reward2, ...). + weights = self.reward_weights + if weights is None: + weights = [1.0] * len(reward_component_keys) + elif len(weights) != len(reward_component_keys): + raise ValueError( + f"reward_weights has {len(weights)} entries but this batch has " + f"{len(reward_component_keys)} reward components ({reward_component_keys}). " + "Provide exactly one weight per component, ordered to match reward1, reward2, ..." + ) valid = torch.ones_like(repeated_batch[reward_component_keys[0]]) leave_one_out = self.use_leave_one_out_baseline assert prompt_ids.shape[0] == valid.shape[0], ( @@ -144,7 +157,9 @@ def compute_advantage( advantage_parts.append(adv_k) - advantages = sum(advantage_parts) + advantages = sum( + weight * adv_k for weight, adv_k in zip(weights, advantage_parts) + ) # Normalize combined advantage to zero mean and unit std adv_std = advantages.std() if adv_std > 0: diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index a3a725d0cb6..d822356e014 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -160,6 +160,10 @@ class AdvEstimatorConfig(TypedDict): # GRPO specific normalize_rewards: NotRequired[bool] use_leave_one_out_baseline: NotRequired[bool] + # GDPO specific: optional per-component weights w_n for the aggregation + # A = sum_n w_n * A_n, ordered to match reward1, reward2, ... . Defaults to + # equal weights (all 1.0) when omitted. + reward_weights: NotRequired[list[float] | None] # Reinforce++ specific minus_baseline: NotRequired[bool] diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 9cc45c1ee36..2276cbc1664 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -469,6 +469,23 @@ def global_post_process_and_metrics(self, batch): raise NotImplementedError +def extract_reward_components(nemo_gym_result: dict) -> Dict[str, float] | None: + """Return per-component rewards from a NeMo Gym verify result, or None. + + Single-reward NeMo Gym environments return only a scalar ``reward``. Multi-reward + environments additionally return ``reward_components``: a mapping of + component-name -> score. These are consumed by GDPO as reward1, reward2, ... + (see ``nemo_rl.algorithms.advantage_estimator.GDPOAdvantageEstimator``). + + Returns ``None`` when the environment is single-reward (no ``reward_components``), + so callers fall back to the scalar ``reward`` path unchanged. + """ + components = nemo_gym_result.get("reward_components") + if not components: + return None + return {str(name): float(score) for name, score in components.items()} + + ######################################## # Global config utils ######################################## diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index 379f544dde4..94906ebbe20 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -2052,6 +2052,34 @@ def run_async_nemo_gym_rollout( if _get_reward_penalty_config_value(resolved_reward_penalty_config, flag): rollout_metrics[metric_name] = penalty_counts[key] / len(results) + # Expose per-component rewards (reward1, reward2, ...) for multi-reward NeMo Gym + # environments so GDPO can compute per-component advantages; single-reward envs are + # unaffected. Mirrors the native rollout path's reward-component handling above. + from nemo_rl.environments.nemo_gym import extract_reward_components + + component_dicts = [extract_reward_components(r["full_result"]) for r in results] + if any(c is not None for c in component_dicts): + # Stable, global component ordering so reward1/reward2/... map to the same + # signal across every sample (components absent on a sample default to 0.0). + ordered_names = sorted( + {name for c in component_dicts if c is not None for name in c} + ) + for component_idx, name in enumerate(ordered_names, start=1): + final_batch[f"reward{component_idx}"] = torch.tensor( + [ + c[name] if c is not None and name in c else 0.0 + for c in component_dicts + ] + ) + # Keep total_reward consistent with the native multi-reward path (sum of + # components) so GDPO and a GRPO control read the same aggregate. + final_batch["total_reward"] = torch.tensor( + [ + sum(c.values()) if c is not None else float(r["full_result"]["reward"]) + for c, r in zip(component_dicts, results) + ] + ) + return AsyncNemoGymRolloutResult( input_ids=input_ids, final_batch=final_batch, diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index df91ba56a0f..ec8377d0399 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -2861,6 +2861,36 @@ def test_gdpo_advantage_estimator_single_reward(): estimator.compute_advantage(prompt_ids, None, mask, repeated_batch) +def test_gdpo_advantage_estimator_reward_weights(): + """GDPO per-reward weights: uniform weights match the default; non-uniform differ; wrong length raises.""" + loss_config = ClippedPGLossConfig() + prompt_ids = torch.tensor([[0], [0], [0]]) + mask = torch.ones(3, 2) + repeated_batch = { + "reward1": torch.tensor([1.0, 0.0, 1.0]), + "reward2": torch.tensor([1.0, 1.0, 0.0]), + } + + def run(weights): + config = {"use_leave_one_out_baseline": False, "normalize_rewards": True} + if weights is not None: + config["reward_weights"] = weights + estimator = GDPOAdvantageEstimator(config, loss_config) + return estimator.compute_advantage(prompt_ids, None, mask, dict(repeated_batch)) + + default = run(None) + + # Any positive uniform scaling is invariant after the final per-batch normalization. + assert torch.allclose(default, run([2.0, 2.0]), atol=1e-5) + + # Non-uniform weights change the advantages. + assert not torch.allclose(default, run([1.0, 0.25]), atol=1e-3) + + # Wrong number of weights -> ValueError. + with pytest.raises(ValueError): + run([1.0]) + + # ============================================================================ # Tests for ReinforcePlusPlusAdvantageEstimator class # ============================================================================ diff --git a/tests/unit/environments/test_nemo_gym.py b/tests/unit/environments/test_nemo_gym.py index 40ceede3d51..b3d64fb629f 100644 --- a/tests/unit/environments/test_nemo_gym.py +++ b/tests/unit/environments/test_nemo_gym.py @@ -29,6 +29,7 @@ from nemo_rl.environments.nemo_gym import ( NemoGym, NemoGymConfig, + extract_reward_components, setup_nemo_gym_config, ) from nemo_rl.models.generation.vllm import VllmGeneration @@ -43,6 +44,23 @@ ) +def test_extract_reward_components(): + """The GDPO multi-reward bridge helper: None for single-reward, normalized dict otherwise.""" + # Single-reward result (no reward_components) -> None, so callers use scalar reward. + assert extract_reward_components({"reward": 1.0}) is None + assert extract_reward_components({"reward": 1.0, "reward_components": {}}) is None + + # Multi-reward result -> name->float dict (values coerced to float, keys to str). + components = extract_reward_components( + { + "reward": 2.0, + "reward_components": {"correctness": 1, "format": 0.5}, + } + ) + assert components == {"correctness": 1.0, "format": 0.5} + assert all(isinstance(v, float) for v in components.values()) + + @pytest.mark.nemo_gym def test_nemo_gym_stub_module(): from nemo_gym import config_types From 19da816a8776f58d721204ea506d2682deb12df5 Mon Sep 17 00:00:00 2001 From: Anjali Shah Date: Thu, 4 Jun 2026 18:24:57 -0700 Subject: [PATCH 2/6] docs: point gdpo_multireward recipe at the example_tool_call_multireward env Name the ready-made multi-reward reference env (NVIDIA-NeMo/Gym) so users know exactly what to set env.nemo_gym to for an end-to-end GDPO multi-reward run. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Anjali Shah --- examples/nemo_gym/gdpo_multireward.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/nemo_gym/gdpo_multireward.yaml b/examples/nemo_gym/gdpo_multireward.yaml index 5911d1812f7..f0f9e69973b 100644 --- a/examples/nemo_gym/gdpo_multireward.yaml +++ b/examples/nemo_gym/gdpo_multireward.yaml @@ -10,7 +10,9 @@ # # NOTE: workplace_assistant is single-reward today; point `env.nemo_gym` at a # multi-reward environment (one whose verifier emits `reward_components`) before -# training, or GDPO will raise "GDPO requires multiple reward components". See +# training, or GDPO will raise "GDPO requires multiple reward components". A ready +# reference env is `example_tool_call_multireward` in NVIDIA-NeMo/Gym (scored on +# correctness / schema_valid / format). See # skills/nemo-rl-gdpo-experiment/references/gym-multireward-scope.md. defaults: grpo_workplace_assistant_nemotron_nano_v2_9b.yaml From fbef9f98a4969b8fe45d5e2cc9ae30a1793dc902 Mon Sep 17 00:00:00 2001 From: Anjali Shah Date: Wed, 15 Jul 2026 16:17:06 -0700 Subject: [PATCH 3/6] fix: don't overwrite total_reward in NeMo Gym multireward path; validate reward == sum(components) Addresses review on #2680: - Stop silently overwriting final_batch["total_reward"] with the component sum. Keep the verifier's scalar `reward`. When reward_components is present the contract is reward == sum(components), so overwriting is a no-op in the correct case and only masks a misconfigured verifier otherwise. Validate the contract instead and raise on a real mismatch. - Expand the sorted() comment to explain it is a canonical ordering (stable reward{n} -> component mapping for the GDPO per-component baseline and reward_weights), not just deduplication. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Anjali Shah --- nemo_rl/experience/rollouts.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index 94906ebbe20..f461a3553c5 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -2059,8 +2059,13 @@ def run_async_nemo_gym_rollout( component_dicts = [extract_reward_components(r["full_result"]) for r in results] if any(c is not None for c in component_dicts): - # Stable, global component ordering so reward1/reward2/... map to the same - # signal across every sample (components absent on a sample default to 0.0). + # Canonical (alphabetical) component ordering, not just deduplication: the + # reward{n} index -> component-name mapping must be stable across every sample + # and every run. GDPO's per-component baseline (calculate_baseline_and_std_per_prompt) + # assumes reward1 denotes the same signal for all responses to a prompt, and + # grpo.adv_estimator.reward_weights is documented as "ordered to match reward1, + # reward2, ...". A first-seen order (e.g. dict.fromkeys) would be batch-dependent + # and could shift that mapping between runs; sorting keeps it deterministic. ordered_names = sorted( {name for c in component_dicts if c is not None for name in c} ) @@ -2071,14 +2076,23 @@ def run_async_nemo_gym_rollout( for c in component_dicts ] ) - # Keep total_reward consistent with the native multi-reward path (sum of - # components) so GDPO and a GRPO control read the same aggregate. - final_batch["total_reward"] = torch.tensor( - [ - sum(c.values()) if c is not None else float(r["full_result"]["reward"]) - for c, r in zip(component_dicts, results) - ] - ) + # Leave total_reward as the verifier's scalar `reward` (set above); do not + # silently overwrite it. When a verifier emits reward_components, the contract is + # reward == sum(components), so overwriting would be a no-op in the correct case + # and would only mask a misconfigured verifier when it isn't. Validate that + # contract instead and fail fast on a real mismatch. + for idx, (components, r) in enumerate(zip(component_dicts, results)): + if components is None: + continue + scalar_reward = float(r["full_result"]["reward"]) + component_sum = sum(components.values()) + if not math.isclose(scalar_reward, component_sum, rel_tol=1e-5, abs_tol=1e-6): + raise ValueError( + f"NeMo Gym verify result {idx} has reward={scalar_reward} but its " + f"reward_components sum to {component_sum} ({components}). A multi-reward " + "verifier must set reward = sum(reward_components.values()) so single-reward " + "(GRPO) consumers and GDPO read the same aggregate." + ) return AsyncNemoGymRolloutResult( input_ids=input_ids, From de41394efc7084523e67738f1abed96625463985 Mon Sep 17 00:00:00 2001 From: Anjali Shah Date: Wed, 15 Jul 2026 16:20:58 -0700 Subject: [PATCH 4/6] test: cover reward==sum(components) validation; extract to helper - Move the reward/component-sum consistency check into validate_reward_components_match_scalar() in nemo_gym.py, next to extract_reward_components, so it is unit-testable without the full rollout machinery. run_async_nemo_gym_rollout now calls the helper. - Add test_validate_reward_components_match_scalar covering the matching, float-tolerance, single-reward-skip, and mismatch-raises cases. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Anjali Shah --- nemo_rl/environments/nemo_gym.py | 28 +++++++++++++++++++ nemo_rl/experience/rollouts.py | 18 ++++-------- tests/unit/environments/test_nemo_gym.py | 35 ++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 2276cbc1664..f3161f68913 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import math import os import subprocess from pathlib import Path @@ -486,6 +487,33 @@ def extract_reward_components(nemo_gym_result: dict) -> Dict[str, float] | None: return {str(name): float(score) for name, score in components.items()} +def validate_reward_components_match_scalar(nemo_gym_results: List[dict]) -> None: + """Assert each multi-reward result sets ``reward == sum(reward_components)``. + + A multi-reward verifier must set the scalar ``reward`` to the sum of its + ``reward_components`` so single-reward (GRPO) consumers and GDPO read the same + aggregate. We keep the verifier's scalar ``reward`` as ``total_reward`` rather than + silently overwriting it with the component sum, so a verifier that violates this + contract must be surfaced here instead of masked. + + Raises ``ValueError`` on the first violating result. A no-op for single-reward + results (those without ``reward_components``). + """ + for idx, result in enumerate(nemo_gym_results): + components = extract_reward_components(result) + if components is None: + continue + scalar_reward = float(result["reward"]) + component_sum = sum(components.values()) + if not math.isclose(scalar_reward, component_sum, rel_tol=1e-5, abs_tol=1e-6): + raise ValueError( + f"NeMo Gym verify result {idx} has reward={scalar_reward} but its " + f"reward_components sum to {component_sum} ({components}). A multi-reward " + "verifier must set reward = sum(reward_components.values()) so single-reward " + "(GRPO) consumers and GDPO read the same aggregate." + ) + + ######################################## # Global config utils ######################################## diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index f461a3553c5..c7730057e0a 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -2055,7 +2055,10 @@ def run_async_nemo_gym_rollout( # Expose per-component rewards (reward1, reward2, ...) for multi-reward NeMo Gym # environments so GDPO can compute per-component advantages; single-reward envs are # unaffected. Mirrors the native rollout path's reward-component handling above. - from nemo_rl.environments.nemo_gym import extract_reward_components + from nemo_rl.environments.nemo_gym import ( + extract_reward_components, + validate_reward_components_match_scalar, + ) component_dicts = [extract_reward_components(r["full_result"]) for r in results] if any(c is not None for c in component_dicts): @@ -2081,18 +2084,7 @@ def run_async_nemo_gym_rollout( # reward == sum(components), so overwriting would be a no-op in the correct case # and would only mask a misconfigured verifier when it isn't. Validate that # contract instead and fail fast on a real mismatch. - for idx, (components, r) in enumerate(zip(component_dicts, results)): - if components is None: - continue - scalar_reward = float(r["full_result"]["reward"]) - component_sum = sum(components.values()) - if not math.isclose(scalar_reward, component_sum, rel_tol=1e-5, abs_tol=1e-6): - raise ValueError( - f"NeMo Gym verify result {idx} has reward={scalar_reward} but its " - f"reward_components sum to {component_sum} ({components}). A multi-reward " - "verifier must set reward = sum(reward_components.values()) so single-reward " - "(GRPO) consumers and GDPO read the same aggregate." - ) + validate_reward_components_match_scalar([r["full_result"] for r in results]) return AsyncNemoGymRolloutResult( input_ids=input_ids, diff --git a/tests/unit/environments/test_nemo_gym.py b/tests/unit/environments/test_nemo_gym.py index b3d64fb629f..e5dbf1365a3 100644 --- a/tests/unit/environments/test_nemo_gym.py +++ b/tests/unit/environments/test_nemo_gym.py @@ -31,6 +31,7 @@ NemoGymConfig, extract_reward_components, setup_nemo_gym_config, + validate_reward_components_match_scalar, ) from nemo_rl.models.generation.vllm import VllmGeneration @@ -61,6 +62,40 @@ def test_extract_reward_components(): assert all(isinstance(v, float) for v in components.values()) +def test_validate_reward_components_match_scalar(): + """Multi-reward verifiers must set reward == sum(reward_components); mismatch raises.""" + # Contract satisfied: reward equals the component sum -> no error. + validate_reward_components_match_scalar( + [{"reward": 1.5, "reward_components": {"correctness": 1.0, "format": 0.5}}] + ) + # Float tolerance: tiny rounding differences are accepted. + validate_reward_components_match_scalar( + [ + { + "reward": 1.5000001, + "reward_components": {"correctness": 1.0, "format": 0.5}, + } + ] + ) + # Single-reward results (no reward_components) are skipped entirely. + validate_reward_components_match_scalar([{"reward": 2.0}]) + + # Mismatch (scalar reward != component sum) -> ValueError naming the offending index. + with pytest.raises(ValueError, match="result 1"): + validate_reward_components_match_scalar( + [ + { + "reward": 1.5, + "reward_components": {"correctness": 1.0, "format": 0.5}, + }, + { + "reward": 2.0, + "reward_components": {"correctness": 1.0, "format": 0.5}, + }, + ] + ) + + @pytest.mark.nemo_gym def test_nemo_gym_stub_module(): from nemo_gym import config_types From d00b4316ac30309c46a2cb81f0a9c5faad22947b Mon Sep 17 00:00:00 2001 From: Anjali Shah Date: Mon, 20 Jul 2026 14:19:26 -0700 Subject: [PATCH 5/6] fix: emit reward/ keys from NeMo Gym GDPO bridge (was reward1/reward2) main's get_gdpo_reward_component_keys() selects batch keys by `k.startswith("reward/")`, but the NeMo Gym bridge emitted positional `reward1, reward2, ...`. Those never match, so GDPO saw zero components and raised "GDPO requires multiple reward components". Emit `reward/` instead, matching the native multi-reward path. - rollouts.py: emit final_batch[f"reward/{name}"] (component name carries identity; get_gdpo_reward_component_keys sorts, so no positional index). - test_grpo.py: reward_weights test now uses reward/correctness, reward/format. - Reword reward_weights ordering from "reward1, reward2, ..." to "alphabetical by component name" across advantage_estimator.py, grpo.py, grpo.md, and gdpo_multireward.yaml; update nemo_gym.py docstring. - Remove dead private-fork link (gym-multireward-scope.md) from the recipe yaml. Addresses review feedback on #2680. Signed-off-by: Anjali Shah --- docs/guides/grpo.md | 5 +++-- examples/nemo_gym/gdpo_multireward.yaml | 7 +++--- nemo_rl/algorithms/advantage_estimator.py | 6 +++-- nemo_rl/algorithms/grpo.py | 4 ++-- nemo_rl/environments/nemo_gym.py | 4 ++-- nemo_rl/experience/rollouts.py | 27 ++++++++++++----------- tests/unit/algorithms/test_grpo.py | 4 ++-- 7 files changed, 30 insertions(+), 27 deletions(-) diff --git a/docs/guides/grpo.md b/docs/guides/grpo.md index 431966b8735..13629d7ff8b 100755 --- a/docs/guides/grpo.md +++ b/docs/guides/grpo.md @@ -637,8 +637,9 @@ Note that this method only has an effect when training involves more than one re The aggregation weights \\( w_n \\) in \\( A^{(i,j)} = \sum_{n=1}^{N} w_n A_n^{(i,j)} \\) are configurable via `reward_weights`, an optional list with one entry per reward -component, ordered to match `reward1, reward2, ...`. When omitted, all weights default -to `1.0` (equal weighting). For example, to down-weight the second and third rewards: +component, ordered alphabetically by component name (matching the sorted `reward/` +keys). When omitted, all weights default to `1.0` (equal weighting). For example, to +down-weight the second and third rewards: ``` grpo: adv_estimator: diff --git a/examples/nemo_gym/gdpo_multireward.yaml b/examples/nemo_gym/gdpo_multireward.yaml index f0f9e69973b..6325516e3db 100644 --- a/examples/nemo_gym/gdpo_multireward.yaml +++ b/examples/nemo_gym/gdpo_multireward.yaml @@ -5,15 +5,14 @@ # components: its verify endpoint must include `reward_components` (a mapping of # component-name -> score) in addition to the scalar `reward`. The Gym->NeMo-RL bridge # (nemo_rl/environments/nemo_gym.py::extract_reward_components, surfaced in -# run_async_nemo_gym_rollout) then exposes them as reward1, reward2, ... for the GDPO +# run_async_nemo_gym_rollout) then exposes them as reward/ keys for the GDPO # advantage estimator. # # NOTE: workplace_assistant is single-reward today; point `env.nemo_gym` at a # multi-reward environment (one whose verifier emits `reward_components`) before # training, or GDPO will raise "GDPO requires multiple reward components". A ready # reference env is `example_tool_call_multireward` in NVIDIA-NeMo/Gym (scored on -# correctness / schema_valid / format). See -# skills/nemo-rl-gdpo-experiment/references/gym-multireward-scope.md. +# correctness / schema_valid / format). defaults: grpo_workplace_assistant_nemotron_nano_v2_9b.yaml grpo: @@ -21,7 +20,7 @@ grpo: name: "gdpo" normalize_rewards: true use_leave_one_out_baseline: false - # Optional per-component weights, ordered to match reward1, reward2, ... . + # Optional per-component weights, ordered alphabetically by component name. # Omit for equal weighting; uncomment and size to your env's components to tune. # reward_weights: [1.0, 0.5, 0.25] diff --git a/nemo_rl/algorithms/advantage_estimator.py b/nemo_rl/algorithms/advantage_estimator.py index 0ab81a80f72..92df9b0fa49 100644 --- a/nemo_rl/algorithms/advantage_estimator.py +++ b/nemo_rl/algorithms/advantage_estimator.py @@ -122,7 +122,8 @@ def compute_advantage( f"This batch has {len(reward_component_keys)} component(s): {reward_component_keys}. " "Switch to GRPO by setting grpo.adv_estimator.name to 'grpo' in your config." ) - # Resolve per-reward weights (ordered to match reward1, reward2, ...). + # Resolve per-reward weights (ordered to match the reward/ components, + # sorted alphabetically by name — same order as reward_component_keys). weights = self.reward_weights if weights is None: weights = [1.0] * len(reward_component_keys) @@ -130,7 +131,8 @@ def compute_advantage( raise ValueError( f"reward_weights has {len(weights)} entries but this batch has " f"{len(reward_component_keys)} reward components ({reward_component_keys}). " - "Provide exactly one weight per component, ordered to match reward1, reward2, ..." + "Provide exactly one weight per component, ordered alphabetically by " + "component name (matching the sorted reward/ keys)." ) valid = torch.ones_like(repeated_batch[reward_component_keys[0]]) leave_one_out = self.use_leave_one_out_baseline diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index d822356e014..c3acf8f11a4 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -161,8 +161,8 @@ class AdvEstimatorConfig(TypedDict): normalize_rewards: NotRequired[bool] use_leave_one_out_baseline: NotRequired[bool] # GDPO specific: optional per-component weights w_n for the aggregation - # A = sum_n w_n * A_n, ordered to match reward1, reward2, ... . Defaults to - # equal weights (all 1.0) when omitted. + # A = sum_n w_n * A_n, ordered alphabetically by component name (matching the sorted + # reward/ keys). Defaults to equal weights (all 1.0) when omitted. reward_weights: NotRequired[list[float] | None] # Reinforce++ specific minus_baseline: NotRequired[bool] diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index f3161f68913..d26010f7266 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -475,8 +475,8 @@ def extract_reward_components(nemo_gym_result: dict) -> Dict[str, float] | None: Single-reward NeMo Gym environments return only a scalar ``reward``. Multi-reward environments additionally return ``reward_components``: a mapping of - component-name -> score. These are consumed by GDPO as reward1, reward2, ... - (see ``nemo_rl.algorithms.advantage_estimator.GDPOAdvantageEstimator``). + component-name -> score. These are surfaced as ``reward/`` batch keys and + consumed by GDPO (see ``nemo_rl.algorithms.advantage_estimator.GDPOAdvantageEstimator``). Returns ``None`` when the environment is single-reward (no ``reward_components``), so callers fall back to the scalar ``reward`` path unchanged. diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index c7730057e0a..56ee0eef414 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -2052,9 +2052,9 @@ def run_async_nemo_gym_rollout( if _get_reward_penalty_config_value(resolved_reward_penalty_config, flag): rollout_metrics[metric_name] = penalty_counts[key] / len(results) - # Expose per-component rewards (reward1, reward2, ...) for multi-reward NeMo Gym - # environments so GDPO can compute per-component advantages; single-reward envs are - # unaffected. Mirrors the native rollout path's reward-component handling above. + # Expose per-component rewards as `reward/` batch keys for multi-reward NeMo + # Gym environments so GDPO can compute per-component advantages; single-reward envs + # are unaffected. Mirrors the native rollout path's reward-component handling above. from nemo_rl.environments.nemo_gym import ( extract_reward_components, validate_reward_components_match_scalar, @@ -2062,18 +2062,19 @@ def run_async_nemo_gym_rollout( component_dicts = [extract_reward_components(r["full_result"]) for r in results] if any(c is not None for c in component_dicts): - # Canonical (alphabetical) component ordering, not just deduplication: the - # reward{n} index -> component-name mapping must be stable across every sample - # and every run. GDPO's per-component baseline (calculate_baseline_and_std_per_prompt) - # assumes reward1 denotes the same signal for all responses to a prompt, and - # grpo.adv_estimator.reward_weights is documented as "ordered to match reward1, - # reward2, ...". A first-seen order (e.g. dict.fromkeys) would be batch-dependent - # and could shift that mapping between runs; sorting keeps it deterministic. - ordered_names = sorted( + # Emit each component under a `reward/` key, matching the native + # multi-reward path and what get_gdpo_reward_component_keys() consumes (it selects + # keys starting with "reward/" and sorts them by name). The name carries the + # component identity, so ordering is handled downstream by that sort — no + # positional index needed. Take the union of names across the batch and default a + # component absent on a given sample to 0.0, so every sample carries the same key + # set (the per-prompt baseline requires each reward/ present for all + # responses to a prompt). + component_names = sorted( {name for c in component_dicts if c is not None for name in c} ) - for component_idx, name in enumerate(ordered_names, start=1): - final_batch[f"reward{component_idx}"] = torch.tensor( + for name in component_names: + final_batch[f"reward/{name}"] = torch.tensor( [ c[name] if c is not None and name in c else 0.0 for c in component_dicts diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index ec8377d0399..a5eaff9943e 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -2867,8 +2867,8 @@ def test_gdpo_advantage_estimator_reward_weights(): prompt_ids = torch.tensor([[0], [0], [0]]) mask = torch.ones(3, 2) repeated_batch = { - "reward1": torch.tensor([1.0, 0.0, 1.0]), - "reward2": torch.tensor([1.0, 1.0, 0.0]), + "reward/correctness": torch.tensor([1.0, 0.0, 1.0]), + "reward/format": torch.tensor([1.0, 1.0, 0.0]), } def run(weights): From 9143bf21584d96f97cb81c5967f4e9f5f9ad4c59 Mon Sep 17 00:00:00 2001 From: Anjali Shah Date: Mon, 20 Jul 2026 14:37:21 -0700 Subject: [PATCH 6/6] test: add GDPO per-reward-weights nightly test Guards the reward_weights aggregation path added to GDPOAdvantageEstimator (per review on #2680). Adds a DTensor GDPO recipe over the native math_multi_reward env (correctness / format / integer components), Qwen2.5-1.5B on GSM8K, 1n8g, with reward_weights=[1.0, 0.5, 0.25]. Follows the nightly pattern from #1866: - examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml - tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh - register in tests/test_suites/nightly.txt - add "gdpo" -> gdpo_math_1B.yaml to ALGO_MAPPING_TO_BASE_YAML so the recipe name-prefix check passes. Signed-off-by: Anjali Shah --- ...wen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml | 20 +++++++++ ...-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh | 44 +++++++++++++++++++ tests/test_suites/nightly.txt | 9 ++++ tests/unit/test_recipes_and_test_suites.py | 1 + 4 files changed, 74 insertions(+) create mode 100644 examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml create mode 100755 tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh diff --git a/examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml b/examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml new file mode 100644 index 00000000000..b76f8172e20 --- /dev/null +++ b/examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml @@ -0,0 +1,20 @@ +defaults: ../../gdpo_math_1B.yaml +grpo: + num_prompts_per_step: 8 + num_generations_per_prompt: 8 + adv_estimator: + reward_weights: + - 1.0 + - 0.5 + - 0.25 +checkpointing: + checkpoint_dir: results/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights +policy: + train_global_batch_size: 64 +logger: + tensorboard_enabled: true + wandb: + project: nemo-rl + name: gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights +cluster: + gpus_per_node: 8 diff --git a/tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh b/tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh new file mode 100755 index 00000000000..08cf1e81b7d --- /dev/null +++ b/tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh @@ -0,0 +1,44 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=1 +GPUS_PER_NODE=8 +STEPS_PER_RUN=10 +MAX_STEPS=10 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +NUM_MINUTES=30 +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +# Run the experiment +cd $PROJECT_ROOT +uv run examples/run_grpo.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=nemo-rl \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + $@ \ + 2>&1 | tee $RUN_LOG + +# Convert tensorboard logs to json +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Only run metrics if the target step is reached +if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then + uv run tests/check_metrics.py $JSON_METRICS \ + 'mean(data["train/gen_kl_error"]) < 0.02' \ + 'max(data["train/reward"]) > 0.05' \ + 'mean(data["timing/train/total_step_time"], -6, -1) < 180' + + # Clean up checkpoint directory after successful run to save space. + rm -rf "$CKPT_DIR" +fi diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index 81e8ebc90c4..56fadfb12a8 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -269,3 +269,12 @@ tests/test_suites/llm/ppo-qwen2.5-1.5b-gsm8k-1n8g-megatron-valuetp2sp-dynbatch.s # Megatron PPO with value SP + pipeline + context parallelism + sequence packing (TP2+SP+PP2+CP2) (Qwen2.5-1.5B, GSM8K) tests/test_suites/llm/ppo-qwen2.5-1.5b-gsm8k-1n8g-megatron-valuetp2sp-pp2cp2-pack.sh + +######## +# GDPO # +######## + +# DTensor GDPO with per-reward weights on the native math_multi_reward env +# (correctness / format / integer), Qwen2.5-1.5B, GSM8K — guards the reward_weights +# aggregation path in GDPOAdvantageEstimator. +tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh diff --git a/tests/unit/test_recipes_and_test_suites.py b/tests/unit/test_recipes_and_test_suites.py index 3e7f01365cc..911eaf1070b 100644 --- a/tests/unit/test_recipes_and_test_suites.py +++ b/tests/unit/test_recipes_and_test_suites.py @@ -48,6 +48,7 @@ "prorlv2": "examples/configs/prorlv2.v2.yaml", "ppo": "examples/configs/ppo_math_1B_megatron.yaml", "mopd": "examples/configs/grpo_math_1B.yaml", + "gdpo": "examples/configs/gdpo_math_1B.yaml", } # Configuration keys that are allowed to be added to base configs during testing