Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
8ae06df
Turn on training cudagraphs for RL
mathemakitten Dec 1, 2025
97266e3
Add data pointer comparison to tensors in ArgMetadata
mathemakitten Dec 2, 2025
8cd1808
Merge branch 'main' into helenn-training-graphs-rl
mathemakitten Dec 2, 2025
e59966f
persist cudagraphs
mathemakitten Dec 5, 2025
836bda6
Gate training cudagraph persistency via -–rl-persist-cuda-graphs
mathemakitten Dec 8, 2025
c8903d9
Fix typo in args
mathemakitten Dec 8, 2025
4185aac
persist cudagraphs
mathemakitten Dec 8, 2025
b966303
Update to rl_training_cuda_graphs
mathemakitten Dec 9, 2025
fd156c3
Merge main
mathemakitten Dec 9, 2025
587ee8c
Fix RNG state tracking for cudagraphs and grad mode
mathemakitten Dec 15, 2025
484d916
Include Peter's change on RNG state tracking safety for cudagraphs
mathemakitten Dec 15, 2025
b271f28
Include Peter's change on RNG state tracking safety for cudagraphs
mathemakitten Dec 15, 2025
cae2cb9
merge main
mathemakitten Jan 12, 2026
52600fb
Merge
mathemakitten Jan 12, 2026
075530f
Flip training cudagraphs back on after refit MR
mathemakitten Jan 12, 2026
3e6fafc
Fix RL sequence packing bin size (#2909)
tdene Jan 12, 2026
c0882a4
If empty bin, give it a default PackedSeqParams so the signature will…
mathemakitten Jan 12, 2026
3d71442
runs fast need refactor
mathemakitten Jan 13, 2026
67b9f3b
Cleanup
mathemakitten Jan 13, 2026
792310a
Merge branch 'main' into rl-training-graphs
mathemakitten Jan 13, 2026
db1bae4
Maximize --rl-sequence-packing-max-sequences-per-bin for now
mathemakitten Jan 13, 2026
fb687b0
Merge branch 'main' into rl-training-graphs
mathemakitten Jan 14, 2026
1c488ff
Scope the self.training check so logprobs will not trigger cudagraph …
mathemakitten Jan 14, 2026
d37d0ce
Add an error if we ever pass a set of tensors through _clone_nested_t…
mathemakitten Jan 14, 2026
69fdba5
Merge branch 'main' into helenn-training-graphs-rl
mathemakitten Jan 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions megatron/core/inference/contexts/dynamic_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ def __init__(
cuda_graph_mixed_prefill_count: Optional[int] = 16,
metrics_writer: Optional['WandbModule'] = None,
request_metadata_types: Optional[List[Tuple[str, torch.dtype, bool]]] = None,
persist_cuda_graphs: Optional[bool] = False,
):
super().__init__(materialize_only_last_token_logits=materialize_only_last_token_logits)

Expand Down Expand Up @@ -362,6 +363,7 @@ def __init__(

# Unified memory.
self.unified_memory_level = unified_memory_level
self.persist_cuda_graphs = persist_cuda_graphs
if unified_memory_level > 0:
try:
self.unified_memory_mempool = create_unified_mempool()
Expand Down
11 changes: 6 additions & 5 deletions megatron/core/inference/engines/dynamic_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ def __init__(
self.enable_chunked_prefill = enable_chunked_prefill
self.inference_logging_step_interval = inference_logging_step_interval
self.unified_memory_level = context.unified_memory_level
self.persist_cuda_graphs = context.persist_cuda_graphs

if enable_cuda_graph is not None:
self.cuda_graph_impl = "local" if enable_cuda_graph else "none"
Expand Down Expand Up @@ -564,10 +565,10 @@ def suspend(self):
):
self.context.deallocate_all_tensors()

# Delete cuda graphs when not using unified memory at all (level 0). For
# levels 1 and 2, the context's tensors maintain static memory addresses,
# so the cuda graphs are re-used.
if self.unified_memory_level == 0:
# Delete cuda graphs when not using unified memory at all (level 0) and
# `--rl-training-cuda-graphs` is not passed. For UVM levels 1 and 2, the context's tensors
# maintain static memory addresses, so the cuda graphs are re-used.
if self.unified_memory_level == 0 and not self.persist_cuda_graphs:
delete_cuda_graphs()

# Maintain references to requests before reset.
Expand Down Expand Up @@ -609,7 +610,7 @@ def resume(self):
# 0). For levels 1 and 2, the context's tensors maintain static
# memory addresses, so the cuda graphs are re-used.
capture_time = time.time()
if self.unified_memory_level == 0:
if self.unified_memory_level == 0 and not self.persist_cuda_graphs:
self.create_cuda_graphs()
capture_time = time.time() - capture_time

Expand Down
185 changes: 183 additions & 2 deletions megatron/core/transformer/cuda_graphs.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def __init__(self, arg):
self.shape = arg.shape
self.dtype = arg.dtype
self.device = arg.device
self.value = arg.data_ptr()
else:
self.value = arg

Expand Down Expand Up @@ -174,6 +175,40 @@ def _determine_if_first_last_layer_of_this_vp_chunk(base_module):
)


def _clone_nested_tensors(value: Any) -> Any:
"""Recursively clone tensors inside nested containers."""
if torch.is_tensor(value):
return value.clone()
if isinstance(value, (tuple, list)):
return type(value)(_clone_nested_tensors(v) for v in value)
if isinstance(value, dict):
return {k: _clone_nested_tensors(v) for k, v in value.items()}
return value

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems a bit dangerous to have Any and not throw an exception for unsupported data types. For instance, if I send a set of tensors, this will not clone them but return a set of tensors which a user of the function would expect to have cloned.

@mathemakitten mathemakitten Jan 13, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We actually don't want to throw an exception, but to pass-through for unsupported datatypes. This helper function is scoped very tightly within the cudagraphs code such that the "send set of tensors" case would *not be realistic, in my opinion.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did not think of that! Got it now. The only q I have is why if I send a tuple/list of tensors we want to clone them, but if I send a set of tensors we don't clone them?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sets of tensors are never used as cudagraph input/output containers — it wouldn't make sense because inputs are ordered, duplicates are meaningful, etc. I would be more concerned if we were sending in a set of tensors at all so I will just throw an error.

This function does see other types (e.g. generators) which need to pass through so I'm not going to modify the typing hints.



def _ensure_generator_state_is_cudagraph_safe(gen: torch.Generator) -> torch.Generator:
"""Make generator state safe for CUDA graph capture/replay.

Generator state tensors can become inference tensors if created under `torch.inference_mode()`.
CUDA graph capture may later attempt in-place updates on that state; this fails for inference
tensors. Fix the generator *in-place* (preserving identity) by cloning its state outside
inference mode and setting it back.
"""
with torch.inference_mode(mode=False):
if hasattr(gen, "graphsafe_get_state"):
state = gen.graphsafe_get_state()
else:
state = gen.get_state()

cloned_state = _clone_nested_tensors(state)
if hasattr(gen, "graphsafe_set_state"):
gen.graphsafe_set_state(cloned_state)
else:
gen.set_state(cloned_state)

return gen


class _CudagraphGlobalRecord:
"""A global datastructure that records of the ordering of all _CudaGraphRunner's
first fwd or bwd passes. 'create_cudagraphs' will use this to create
Expand Down Expand Up @@ -680,14 +715,151 @@ def create_fwd_graph(self, args, kwargs, clone_inputs=True):

self.fwd_graph = torch.cuda.CUDAGraph()

# For cases with multiple active RNG states, e.g. TP.
rng_states = get_all_rng_states()
with torch.inference_mode(mode=False):
for gen in rng_states.values():
self.fwd_graph.register_generator_state(
_ensure_generator_state_is_cudagraph_safe(gen)
)

# warmup again as case graph capture mode may execute a different codepath
for _ in range(self.num_warmup_steps):
with self.get_quantization_context():
outputs = self.base_module.forward(*args, **kwargs)
if self.training and torch.is_grad_enabled():
if isinstance(outputs, torch.Tensor):
outputs = (outputs,)
outputs = self.get_tensors(outputs)
grad_inputs = torch.autograd.grad(
outputs=tuple(o for o in outputs if o.requires_grad),
inputs=tuple(i for i in self.fwd_graph_input_surface if i.requires_grad),
grad_outputs=tuple(
torch.zeros_like(o) if o.requires_grad else None for o in outputs
),
only_inputs=True,
allow_unused=True,
)

with self.get_quantization_context():
torch.cuda.synchronize()
# Register default CUDA generators ourselves (fixed in-place to have normal tensors)
# before capture begins, to avoid inference-tensor state issues during capture.
with torch.inference_mode(mode=False):
for device_idx in range(torch.cuda.device_count()):
default_gen = torch.cuda.default_generators[device_idx]
self.fwd_graph.register_generator_state(
_ensure_generator_state_is_cudagraph_safe(default_gen)
)

with torch.cuda.graph(
self.fwd_graph, pool=self.fwd_mempool, capture_error_mode="thread_local"
):
outputs = self.base_module.forward(*args, **kwargs)

# save cudagraph output buffer
if isinstance(outputs, torch.Tensor):
outputs = (outputs,)
self.fwd_graph_outputs = outputs
self.fwd_graph_output_surface = self.get_tensors(outputs)

if self.training and torch.is_grad_enabled():
assert (
len(self.fwd_graph_output_surface) > 0
), """Tried graphing a moudule that returned no tensors in training mode,
however the graphed module must output at least one tensor,
so that a corresponding backward node may be registered in the autograd graph."""

# restore cached grads
for param in self.base_module.parameters():
if hasattr(param, 'main_grad'):
saved_grad = save_main_grads.pop(0)
assert (
param.main_grad.shape == saved_grad.shape
), "Error restoring grads while cudagraphing!"
param.main_grad.copy_(saved_grad)

if self.fp8_enabled or self.fp4_enabled:
restore_fp8_tensors([self.base_module], saved_fp8_tensors)

# Unfreeze GC.
if FREEZE_GC:
gc.unfreeze()

# gc.collect() drops references to unreachable tensors created during capture,
# returning their storage to the allocator to avoid a slowdown during replay. However,
# it forces expensive global garbage collection, so must be done only on the last layer
# per-device to avoid slowing down graph creation.
if self.is_last_layer:
gc.collect()

def create_fwd_graph_old(self, args, kwargs, clone_inputs=True):
"""Create a fwd cudagraph for this runner. Should be called inside
'create_cudagraphs()'."""

# Freeze GC, to speed up capture time ~15-20x.
if FREEZE_GC:
gc.freeze()

# save grads and other variables that may be affected by graph warmup
if self.training and torch.is_grad_enabled():
save_main_grads = [
param.main_grad.clone()
for param in self.base_module.parameters()
if hasattr(param, 'main_grad')
]

saved_fp8_tensors = None

if self.fp8_enabled:
if is_te_min_version("1.13.0"):
saved_fp8_tensors = save_fp8_tensors([self.base_module], self.fp8_recipe)
else:
saved_fp8_tensors = save_fp8_tensors(
[self.base_module], self.fp8_recipe.amax_history_len
)
elif self.fp4_enabled:
if is_te_min_version("2.7.0.dev0"):
saved_fp8_tensors = save_fp8_tensors([self.base_module], self.fp4_recipe)
else:
raise ValueError("FP4 requires TE >= 2.7.0.dev0 for NVFP4BlockScaling support.")

if clone_inputs:
args, kwargs = self.zero_out_tensors(args, kwargs)

input_tensors = self.get_tensors(args, kwargs)
self.fwd_graph_input_surface = input_tensors + tuple(self.base_module.parameters())

self.fwd_graph = torch.cuda.CUDAGraph()

# For cases with multiple active RNG states, e.g. TP.
for _, state in get_all_rng_states().items():
rng_states = get_all_rng_states()
with torch.inference_mode(mode=False):
for gen in rng_states.values():
self.fwd_graph.register_generator_state(
_ensure_generator_state_is_cudagraph_safe(gen)
)
self.fwd_graph.register_generator_state(state)

# warmup again as case graph capture mode may execute a different codepath
for _ in range(self.num_warmup_steps):
with self.get_quantization_context():
outputs = self.base_module.forward(*args, **kwargs)
torch.cuda.synchronize()
# Register default CUDA generators ourselves (fixed in-place to have normal tensors)
# before capture begins, to avoid inference-tensor state issues during capture.
with torch.inference_mode(mode=False):
for device_idx in range(torch.cuda.device_count()):
default_gen = torch.cuda.default_generators[device_idx]
self.fwd_graph.register_generator_state(
_ensure_generator_state_is_cudagraph_safe(default_gen)
)

with torch.cuda.graph(
self.fwd_graph, pool=self.fwd_mempool, capture_error_mode="thread_local"
):
outputs = self.base_module.forward(*args, **kwargs)

if self.training and torch.is_grad_enabled():
if isinstance(outputs, torch.Tensor):
outputs = (outputs,)
Expand All @@ -704,6 +876,15 @@ def create_fwd_graph(self, args, kwargs, clone_inputs=True):

with self.get_quantization_context():
torch.cuda.synchronize()
# Register default CUDA generators ourselves (fixed in-place to have normal tensors)
# before capture begins, to avoid inference-tensor state issues during capture.
with torch.inference_mode(mode=False):
for device_idx in range(torch.cuda.device_count()):
default_gen = torch.cuda.default_generators[device_idx]
self.fwd_graph.register_generator_state(
_ensure_generator_state_is_cudagraph_safe(default_gen)
)

with torch.cuda.graph(
self.fwd_graph, pool=self.fwd_mempool, capture_error_mode="thread_local"
):
Expand Down Expand Up @@ -1319,7 +1500,7 @@ def __call__(self, megatron_module, args, kwargs):
# Now replay the graph
out = runner.replay_graph_capture(self.is_first_microbatch, args, kwargs)

elif self.training:
elif self.training and torch.is_grad_enabled():

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not related specifically to this line or meant to be address in this MR, but do we ever need to be able to distinguish between no_grad and inference_mode in the code? If two forwards have the same signature but one is no grad and another is inference mode I wonder if it will error at all in inlace ops

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've made this unnecessary by flipping to model.eval() and then back to model.train() in logprobs_forward_step so it will never trigger cudagraph capture.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But yes you're right — right now we rely on cudagraph runner matching guardrails like correct shapes and arguments (e.g. inference passes some arguments which training does not). If we ever have graphs with the exact same shapes and arguments but differing only between inference mode and no_grad mode we may have issues.

# Training mode
runner = self.get_cudagraph_runner(megatron_module, args, kwargs)
# check if a layer is frozen during training.
Expand Down
1 change: 1 addition & 0 deletions megatron/rl/inference/megatron.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ def get_dynamic_inference_engine(args: Namespace, model: MegatronModule, inferen
cuda_graph_max_tokens=args.inference_dynamic_batching_cuda_graph_max_tokens,
cuda_graph_mixed_prefill_count=args.inference_dynamic_batching_cuda_graph_mixed_prefill_count,
metrics_writer=metrics_writer,
persist_cuda_graphs=args.rl_training_cuda_graphs
)

inference_wrapped_model = GPTInferenceWrapper(model, args, inference_context)
Expand Down
4 changes: 2 additions & 2 deletions megatron/rl/rl_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2432,7 +2432,7 @@ def megatron_rl_inference_mode(
optimizer.offload_to_cpu()

# TODO: Remove this if statement once a change to `toggle_cuda_graphs` makes it safe to.
if cuda_graph_impl != "none":
if cuda_graph_impl != "none" and not args.rl_training_cuda_graphs:
toggle_cuda_graphs(lang_module, cuda_graph_impl, reset_cuda_graphs=reset_cuda_graphs)

inference_interface = get_inference_interface(args, loop, model)
Expand Down Expand Up @@ -2481,7 +2481,7 @@ def megatron_rl_inference_mode(
inference_interface._inference_engine.context.memory_buffer = None

# TODO: Remove this if statement once a change to `toggle_cuda_graphs` makes it safe to.
if cuda_graph_impl != "none":
if cuda_graph_impl != "none" and not args.rl_training_cuda_graphs:
toggle_cuda_graphs(lang_module, 'none', reset_cuda_graphs=reset_cuda_graphs)

if offload_optimizer_during_inference:
Expand Down
4 changes: 4 additions & 0 deletions megatron/training/arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -2041,6 +2041,10 @@ def _add_rl_args(parser):
help='Algorithm for distributing packed bins across ranks. '
'fifo: first-in-first-out sequential distribution, '
'round-robin: distribute bins cyclically across ranks for better load balancing')
group.add_argument('--rl-training-cuda-graphs', action=argparse.BooleanOptionalAction, type=bool,
default=False,
help='If set, do not call `delete_cuda_graphs` or `toggle_cuda_graphs` when the inference engine is suspended. '
'Use only when all training and inference cudagraphs and the KV cache fit on device.')
return parser

def _add_training_args(parser):
Expand Down
Loading