diff --git a/docs/guides/grpo.md b/docs/guides/grpo.md index 579a0af52a0..13629d7ff8b 100755 --- a/docs/guides/grpo.md +++ b/docs/guides/grpo.md @@ -633,6 +633,23 @@ grpo: ``` Note that this method only has an effect when training involves more than one reward function. +#### Per-reward weights + +The aggregation weights \\( w_n \\) in \\( A^{(i,j)} = \sum_{n=1}^{N} w_n A_n^{(i,j)} \\) +are configurable via `reward_weights`, an optional list with one entry per reward +component, ordered alphabetically by component name (matching the sorted `reward/` +keys). When omitted, all weights default to `1.0` (equal weighting). For example, to +down-weight the second and third rewards: +``` +grpo: + adv_estimator: + name: "gdpo" + normalize_rewards: true + use_leave_one_out_baseline: false + reward_weights: [1.0, 0.5, 0.25] +``` +The number of weights must equal the number of reward components, or a `ValueError` is raised. + ## LoRA Configuration GRPO supports LoRA on both the DTensor and Megatron backends. To enable LoRA on the default DTensor backend: diff --git a/examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml b/examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml new file mode 100644 index 00000000000..b76f8172e20 --- /dev/null +++ b/examples/configs/recipes/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.yaml @@ -0,0 +1,20 @@ +defaults: ../../gdpo_math_1B.yaml +grpo: + num_prompts_per_step: 8 + num_generations_per_prompt: 8 + adv_estimator: + reward_weights: + - 1.0 + - 0.5 + - 0.25 +checkpointing: + checkpoint_dir: results/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights +policy: + train_global_batch_size: 64 +logger: + tensorboard_enabled: true + wandb: + project: nemo-rl + name: gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights +cluster: + gpus_per_node: 8 diff --git a/examples/nemo_gym/gdpo_multireward.yaml b/examples/nemo_gym/gdpo_multireward.yaml new file mode 100644 index 00000000000..6325516e3db --- /dev/null +++ b/examples/nemo_gym/gdpo_multireward.yaml @@ -0,0 +1,33 @@ +# GDPO over a multi-reward NeMo Gym environment. +# +# Inherits the workplace-assistant NeMo Gym recipe and switches only the advantage +# estimator to GDPO. GDPO requires the Gym environment to return multiple reward +# components: its verify endpoint must include `reward_components` (a mapping of +# component-name -> score) in addition to the scalar `reward`. The Gym->NeMo-RL bridge +# (nemo_rl/environments/nemo_gym.py::extract_reward_components, surfaced in +# run_async_nemo_gym_rollout) then exposes them as reward/ keys for the GDPO +# advantage estimator. +# +# NOTE: workplace_assistant is single-reward today; point `env.nemo_gym` at a +# multi-reward environment (one whose verifier emits `reward_components`) before +# training, or GDPO will raise "GDPO requires multiple reward components". A ready +# reference env is `example_tool_call_multireward` in NVIDIA-NeMo/Gym (scored on +# correctness / schema_valid / format). +defaults: grpo_workplace_assistant_nemotron_nano_v2_9b.yaml + +grpo: + adv_estimator: + name: "gdpo" + normalize_rewards: true + use_leave_one_out_baseline: false + # Optional per-component weights, ordered alphabetically by component name. + # Omit for equal weighting; uncomment and size to your env's components to tune. + # reward_weights: [1.0, 0.5, 0.25] + +checkpointing: + checkpoint_dir: "results/gdpo_nemo_gym" + +logger: + wandb: + project: "gdpo-dev" + name: "gdpo-nemo-gym-multireward" diff --git a/nemo_rl/algorithms/advantage_estimator.py b/nemo_rl/algorithms/advantage_estimator.py index 5a73902f329..92df9b0fa49 100644 --- a/nemo_rl/algorithms/advantage_estimator.py +++ b/nemo_rl/algorithms/advantage_estimator.py @@ -16,7 +16,7 @@ This module provides different advantage estimation strategies: - GRPOAdvantageEstimator: Standard GRPO advantage with leave-one-out baseline -- GDPOAdvantageEstimator: Multi-reward GDPO (per-component baselines, sum then normalize) +- GDPOAdvantageEstimator: Multi-reward GDPO (per-component baselines, optional per-reward weights, then normalize) - ReinforcePlusPlusAdvantageEstimator: Reinforce++ with optional baseline subtraction (minus_baseline) and KL penalty in reward - RawRewardAdvantageEstimator: Raw reward as advantage with optional batch normalization (no baseline, no value model) - GeneralizedAdvantageEstimator: Generalized Advantage Estimation (GAE) with temporal bootstrapping @@ -91,6 +91,9 @@ class GDPOAdvantageEstimator: def __init__(self, estimator_config: dict, loss_config: ClippedPGLossConfig): self.use_leave_one_out_baseline = estimator_config["use_leave_one_out_baseline"] self.normalize_rewards = estimator_config["normalize_rewards"] + # Optional per-reward weights w_n for the aggregation A = sum_n w_n * A_n + # (paper: https://arxiv.org/abs/2601.05242). None => equal weights (all 1.0). + self.reward_weights = estimator_config.get("reward_weights") def compute_advantage( self, @@ -119,6 +122,18 @@ def compute_advantage( f"This batch has {len(reward_component_keys)} component(s): {reward_component_keys}. " "Switch to GRPO by setting grpo.adv_estimator.name to 'grpo' in your config." ) + # Resolve per-reward weights (ordered to match the reward/ components, + # sorted alphabetically by name — same order as reward_component_keys). + weights = self.reward_weights + if weights is None: + weights = [1.0] * len(reward_component_keys) + elif len(weights) != len(reward_component_keys): + raise ValueError( + f"reward_weights has {len(weights)} entries but this batch has " + f"{len(reward_component_keys)} reward components ({reward_component_keys}). " + "Provide exactly one weight per component, ordered alphabetically by " + "component name (matching the sorted reward/ keys)." + ) valid = torch.ones_like(repeated_batch[reward_component_keys[0]]) leave_one_out = self.use_leave_one_out_baseline assert prompt_ids.shape[0] == valid.shape[0], ( @@ -144,7 +159,9 @@ def compute_advantage( advantage_parts.append(adv_k) - advantages = sum(advantage_parts) + advantages = sum( + weight * adv_k for weight, adv_k in zip(weights, advantage_parts) + ) # Normalize combined advantage to zero mean and unit std adv_std = advantages.std() if adv_std > 0: diff --git a/nemo_rl/algorithms/grpo.py b/nemo_rl/algorithms/grpo.py index a3a725d0cb6..c3acf8f11a4 100644 --- a/nemo_rl/algorithms/grpo.py +++ b/nemo_rl/algorithms/grpo.py @@ -160,6 +160,10 @@ class AdvEstimatorConfig(TypedDict): # GRPO specific normalize_rewards: NotRequired[bool] use_leave_one_out_baseline: NotRequired[bool] + # GDPO specific: optional per-component weights w_n for the aggregation + # A = sum_n w_n * A_n, ordered alphabetically by component name (matching the sorted + # reward/ keys). Defaults to equal weights (all 1.0) when omitted. + reward_weights: NotRequired[list[float] | None] # Reinforce++ specific minus_baseline: NotRequired[bool] diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index 9cc45c1ee36..d26010f7266 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import math import os import subprocess from pathlib import Path @@ -469,6 +470,50 @@ def global_post_process_and_metrics(self, batch): raise NotImplementedError +def extract_reward_components(nemo_gym_result: dict) -> Dict[str, float] | None: + """Return per-component rewards from a NeMo Gym verify result, or None. + + Single-reward NeMo Gym environments return only a scalar ``reward``. Multi-reward + environments additionally return ``reward_components``: a mapping of + component-name -> score. These are surfaced as ``reward/`` batch keys and + consumed by GDPO (see ``nemo_rl.algorithms.advantage_estimator.GDPOAdvantageEstimator``). + + Returns ``None`` when the environment is single-reward (no ``reward_components``), + so callers fall back to the scalar ``reward`` path unchanged. + """ + components = nemo_gym_result.get("reward_components") + if not components: + return None + return {str(name): float(score) for name, score in components.items()} + + +def validate_reward_components_match_scalar(nemo_gym_results: List[dict]) -> None: + """Assert each multi-reward result sets ``reward == sum(reward_components)``. + + A multi-reward verifier must set the scalar ``reward`` to the sum of its + ``reward_components`` so single-reward (GRPO) consumers and GDPO read the same + aggregate. We keep the verifier's scalar ``reward`` as ``total_reward`` rather than + silently overwriting it with the component sum, so a verifier that violates this + contract must be surfaced here instead of masked. + + Raises ``ValueError`` on the first violating result. A no-op for single-reward + results (those without ``reward_components``). + """ + for idx, result in enumerate(nemo_gym_results): + components = extract_reward_components(result) + if components is None: + continue + scalar_reward = float(result["reward"]) + component_sum = sum(components.values()) + if not math.isclose(scalar_reward, component_sum, rel_tol=1e-5, abs_tol=1e-6): + raise ValueError( + f"NeMo Gym verify result {idx} has reward={scalar_reward} but its " + f"reward_components sum to {component_sum} ({components}). A multi-reward " + "verifier must set reward = sum(reward_components.values()) so single-reward " + "(GRPO) consumers and GDPO read the same aggregate." + ) + + ######################################## # Global config utils ######################################## diff --git a/nemo_rl/experience/rollouts.py b/nemo_rl/experience/rollouts.py index 379f544dde4..56ee0eef414 100644 --- a/nemo_rl/experience/rollouts.py +++ b/nemo_rl/experience/rollouts.py @@ -2052,6 +2052,41 @@ def run_async_nemo_gym_rollout( if _get_reward_penalty_config_value(resolved_reward_penalty_config, flag): rollout_metrics[metric_name] = penalty_counts[key] / len(results) + # Expose per-component rewards as `reward/` batch keys for multi-reward NeMo + # Gym environments so GDPO can compute per-component advantages; single-reward envs + # are unaffected. Mirrors the native rollout path's reward-component handling above. + from nemo_rl.environments.nemo_gym import ( + extract_reward_components, + validate_reward_components_match_scalar, + ) + + component_dicts = [extract_reward_components(r["full_result"]) for r in results] + if any(c is not None for c in component_dicts): + # Emit each component under a `reward/` key, matching the native + # multi-reward path and what get_gdpo_reward_component_keys() consumes (it selects + # keys starting with "reward/" and sorts them by name). The name carries the + # component identity, so ordering is handled downstream by that sort — no + # positional index needed. Take the union of names across the batch and default a + # component absent on a given sample to 0.0, so every sample carries the same key + # set (the per-prompt baseline requires each reward/ present for all + # responses to a prompt). + component_names = sorted( + {name for c in component_dicts if c is not None for name in c} + ) + for name in component_names: + final_batch[f"reward/{name}"] = torch.tensor( + [ + c[name] if c is not None and name in c else 0.0 + for c in component_dicts + ] + ) + # Leave total_reward as the verifier's scalar `reward` (set above); do not + # silently overwrite it. When a verifier emits reward_components, the contract is + # reward == sum(components), so overwriting would be a no-op in the correct case + # and would only mask a misconfigured verifier when it isn't. Validate that + # contract instead and fail fast on a real mismatch. + validate_reward_components_match_scalar([r["full_result"] for r in results]) + return AsyncNemoGymRolloutResult( input_ids=input_ids, final_batch=final_batch, diff --git a/tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh b/tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh new file mode 100755 index 00000000000..08cf1e81b7d --- /dev/null +++ b/tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh @@ -0,0 +1,44 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) +source $SCRIPT_DIR/common.env + +# ===== BEGIN CONFIG ===== +NUM_NODES=1 +GPUS_PER_NODE=8 +STEPS_PER_RUN=10 +MAX_STEPS=10 +NUM_RUNS=$(( (MAX_STEPS + STEPS_PER_RUN - 1) / STEPS_PER_RUN )) # Round up +NUM_MINUTES=30 +# ===== END CONFIG ===== + +exit_if_max_steps_reached + +# Run the experiment +cd $PROJECT_ROOT +uv run examples/run_grpo.py \ + --config $CONFIG_PATH \ + grpo.max_num_steps=$MAX_STEPS \ + logger.log_dir=$LOG_DIR \ + logger.wandb_enabled=True \ + logger.wandb.project=nemo-rl \ + logger.wandb.name=$EXP_NAME \ + logger.monitor_gpus=True \ + logger.tensorboard_enabled=True \ + checkpointing.enabled=True \ + checkpointing.checkpoint_dir=$CKPT_DIR \ + $@ \ + 2>&1 | tee $RUN_LOG + +# Convert tensorboard logs to json +uv run tests/json_dump_tb_logs.py $LOG_DIR --output_path $JSON_METRICS + +# Only run metrics if the target step is reached +if [[ $(jq 'to_entries | .[] | select(.key == "train/loss") | .value | keys | map(tonumber) | max' $JSON_METRICS) -ge $MAX_STEPS ]]; then + uv run tests/check_metrics.py $JSON_METRICS \ + 'mean(data["train/gen_kl_error"]) < 0.02' \ + 'max(data["train/reward"]) > 0.05' \ + 'mean(data["timing/train/total_step_time"], -6, -1) < 180' + + # Clean up checkpoint directory after successful run to save space. + rm -rf "$CKPT_DIR" +fi diff --git a/tests/test_suites/nightly.txt b/tests/test_suites/nightly.txt index 81e8ebc90c4..56fadfb12a8 100644 --- a/tests/test_suites/nightly.txt +++ b/tests/test_suites/nightly.txt @@ -269,3 +269,12 @@ tests/test_suites/llm/ppo-qwen2.5-1.5b-gsm8k-1n8g-megatron-valuetp2sp-dynbatch.s # Megatron PPO with value SP + pipeline + context parallelism + sequence packing (TP2+SP+PP2+CP2) (Qwen2.5-1.5B, GSM8K) tests/test_suites/llm/ppo-qwen2.5-1.5b-gsm8k-1n8g-megatron-valuetp2sp-pp2cp2-pack.sh + +######## +# GDPO # +######## + +# DTensor GDPO with per-reward weights on the native math_multi_reward env +# (correctness / format / integer), Qwen2.5-1.5B, GSM8K — guards the reward_weights +# aggregation path in GDPOAdvantageEstimator. +tests/test_suites/llm/gdpo-qwen2.5-1.5b-1n8g-fsdp2-reward-weights.sh diff --git a/tests/unit/algorithms/test_grpo.py b/tests/unit/algorithms/test_grpo.py index df91ba56a0f..a5eaff9943e 100644 --- a/tests/unit/algorithms/test_grpo.py +++ b/tests/unit/algorithms/test_grpo.py @@ -2861,6 +2861,36 @@ def test_gdpo_advantage_estimator_single_reward(): estimator.compute_advantage(prompt_ids, None, mask, repeated_batch) +def test_gdpo_advantage_estimator_reward_weights(): + """GDPO per-reward weights: uniform weights match the default; non-uniform differ; wrong length raises.""" + loss_config = ClippedPGLossConfig() + prompt_ids = torch.tensor([[0], [0], [0]]) + mask = torch.ones(3, 2) + repeated_batch = { + "reward/correctness": torch.tensor([1.0, 0.0, 1.0]), + "reward/format": torch.tensor([1.0, 1.0, 0.0]), + } + + def run(weights): + config = {"use_leave_one_out_baseline": False, "normalize_rewards": True} + if weights is not None: + config["reward_weights"] = weights + estimator = GDPOAdvantageEstimator(config, loss_config) + return estimator.compute_advantage(prompt_ids, None, mask, dict(repeated_batch)) + + default = run(None) + + # Any positive uniform scaling is invariant after the final per-batch normalization. + assert torch.allclose(default, run([2.0, 2.0]), atol=1e-5) + + # Non-uniform weights change the advantages. + assert not torch.allclose(default, run([1.0, 0.25]), atol=1e-3) + + # Wrong number of weights -> ValueError. + with pytest.raises(ValueError): + run([1.0]) + + # ============================================================================ # Tests for ReinforcePlusPlusAdvantageEstimator class # ============================================================================ diff --git a/tests/unit/environments/test_nemo_gym.py b/tests/unit/environments/test_nemo_gym.py index 40ceede3d51..e5dbf1365a3 100644 --- a/tests/unit/environments/test_nemo_gym.py +++ b/tests/unit/environments/test_nemo_gym.py @@ -29,7 +29,9 @@ from nemo_rl.environments.nemo_gym import ( NemoGym, NemoGymConfig, + extract_reward_components, setup_nemo_gym_config, + validate_reward_components_match_scalar, ) from nemo_rl.models.generation.vllm import VllmGeneration @@ -43,6 +45,57 @@ ) +def test_extract_reward_components(): + """The GDPO multi-reward bridge helper: None for single-reward, normalized dict otherwise.""" + # Single-reward result (no reward_components) -> None, so callers use scalar reward. + assert extract_reward_components({"reward": 1.0}) is None + assert extract_reward_components({"reward": 1.0, "reward_components": {}}) is None + + # Multi-reward result -> name->float dict (values coerced to float, keys to str). + components = extract_reward_components( + { + "reward": 2.0, + "reward_components": {"correctness": 1, "format": 0.5}, + } + ) + assert components == {"correctness": 1.0, "format": 0.5} + assert all(isinstance(v, float) for v in components.values()) + + +def test_validate_reward_components_match_scalar(): + """Multi-reward verifiers must set reward == sum(reward_components); mismatch raises.""" + # Contract satisfied: reward equals the component sum -> no error. + validate_reward_components_match_scalar( + [{"reward": 1.5, "reward_components": {"correctness": 1.0, "format": 0.5}}] + ) + # Float tolerance: tiny rounding differences are accepted. + validate_reward_components_match_scalar( + [ + { + "reward": 1.5000001, + "reward_components": {"correctness": 1.0, "format": 0.5}, + } + ] + ) + # Single-reward results (no reward_components) are skipped entirely. + validate_reward_components_match_scalar([{"reward": 2.0}]) + + # Mismatch (scalar reward != component sum) -> ValueError naming the offending index. + with pytest.raises(ValueError, match="result 1"): + validate_reward_components_match_scalar( + [ + { + "reward": 1.5, + "reward_components": {"correctness": 1.0, "format": 0.5}, + }, + { + "reward": 2.0, + "reward_components": {"correctness": 1.0, "format": 0.5}, + }, + ] + ) + + @pytest.mark.nemo_gym def test_nemo_gym_stub_module(): from nemo_gym import config_types diff --git a/tests/unit/test_recipes_and_test_suites.py b/tests/unit/test_recipes_and_test_suites.py index 3e7f01365cc..911eaf1070b 100644 --- a/tests/unit/test_recipes_and_test_suites.py +++ b/tests/unit/test_recipes_and_test_suites.py @@ -48,6 +48,7 @@ "prorlv2": "examples/configs/prorlv2.v2.yaml", "ppo": "examples/configs/ppo_math_1B_megatron.yaml", "mopd": "examples/configs/grpo_math_1B.yaml", + "gdpo": "examples/configs/gdpo_math_1B.yaml", } # Configuration keys that are allowed to be added to base configs during testing