Repository navigation
Enable training cudagraphs for RL #2452
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 11 commits
8ae06df
97266e3
8cd1808
e59966f
836bda6
c8903d9
4185aac
b966303
fd156c3
587ee8c
484d916
b271f28
cae2cb9
52600fb
075530f
3e6fafc
c0882a4
3d71442
67b9f3b
792310a
db1bae4
fb687b0
1c488ff
d37d0ce
69fdba5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -99,6 +99,7 @@ def __init__(self, arg): | |
| self.shape = arg.shape | ||
| self.dtype = arg.dtype | ||
| self.device = arg.device | ||
| self.value = arg.data_ptr() | ||
| else: | ||
| self.value = arg | ||
|
|
||
|
|
@@ -174,6 +175,40 @@ def _determine_if_first_last_layer_of_this_vp_chunk(base_module): | |
| ) | ||
|
|
||
|
|
||
| def _clone_nested_tensors(value: Any) -> Any: | ||
| """Recursively clone tensors inside nested containers.""" | ||
| if torch.is_tensor(value): | ||
| return value.clone() | ||
| if isinstance(value, (tuple, list)): | ||
| return type(value)(_clone_nested_tensors(v) for v in value) | ||
| if isinstance(value, dict): | ||
| return {k: _clone_nested_tensors(v) for k, v in value.items()} | ||
| return value | ||
|
|
||
|
|
||
| def _ensure_generator_state_is_cudagraph_safe(gen: torch.Generator) -> torch.Generator: | ||
| """Make generator state safe for CUDA graph capture/replay. | ||
|
|
||
| Generator state tensors can become inference tensors if created under `torch.inference_mode()`. | ||
| CUDA graph capture may later attempt in-place updates on that state; this fails for inference | ||
| tensors. Fix the generator *in-place* (preserving identity) by cloning its state outside | ||
| inference mode and setting it back. | ||
| """ | ||
| with torch.inference_mode(mode=False): | ||
| if hasattr(gen, "graphsafe_get_state"): | ||
| state = gen.graphsafe_get_state() | ||
| else: | ||
| state = gen.get_state() | ||
|
|
||
| cloned_state = _clone_nested_tensors(state) | ||
| if hasattr(gen, "graphsafe_set_state"): | ||
| gen.graphsafe_set_state(cloned_state) | ||
| else: | ||
| gen.set_state(cloned_state) | ||
|
|
||
| return gen | ||
|
|
||
|
|
||
| class _CudagraphGlobalRecord: | ||
| """A global datastructure that records of the ordering of all _CudaGraphRunner's | ||
| first fwd or bwd passes. 'create_cudagraphs' will use this to create | ||
|
|
@@ -680,14 +715,151 @@ def create_fwd_graph(self, args, kwargs, clone_inputs=True): | |
|
|
||
| self.fwd_graph = torch.cuda.CUDAGraph() | ||
|
|
||
| # For cases with multiple active RNG states, e.g. TP. | ||
| rng_states = get_all_rng_states() | ||
| with torch.inference_mode(mode=False): | ||
| for gen in rng_states.values(): | ||
| self.fwd_graph.register_generator_state( | ||
| _ensure_generator_state_is_cudagraph_safe(gen) | ||
| ) | ||
|
|
||
| # warmup again as case graph capture mode may execute a different codepath | ||
| for _ in range(self.num_warmup_steps): | ||
| with self.get_quantization_context(): | ||
| outputs = self.base_module.forward(*args, **kwargs) | ||
| if self.training and torch.is_grad_enabled(): | ||
| if isinstance(outputs, torch.Tensor): | ||
| outputs = (outputs,) | ||
| outputs = self.get_tensors(outputs) | ||
| grad_inputs = torch.autograd.grad( | ||
| outputs=tuple(o for o in outputs if o.requires_grad), | ||
| inputs=tuple(i for i in self.fwd_graph_input_surface if i.requires_grad), | ||
| grad_outputs=tuple( | ||
| torch.zeros_like(o) if o.requires_grad else None for o in outputs | ||
| ), | ||
| only_inputs=True, | ||
| allow_unused=True, | ||
| ) | ||
|
|
||
| with self.get_quantization_context(): | ||
| torch.cuda.synchronize() | ||
| # Register default CUDA generators ourselves (fixed in-place to have normal tensors) | ||
| # before capture begins, to avoid inference-tensor state issues during capture. | ||
| with torch.inference_mode(mode=False): | ||
| for device_idx in range(torch.cuda.device_count()): | ||
| default_gen = torch.cuda.default_generators[device_idx] | ||
| self.fwd_graph.register_generator_state( | ||
| _ensure_generator_state_is_cudagraph_safe(default_gen) | ||
| ) | ||
|
|
||
| with torch.cuda.graph( | ||
| self.fwd_graph, pool=self.fwd_mempool, capture_error_mode="thread_local" | ||
| ): | ||
| outputs = self.base_module.forward(*args, **kwargs) | ||
|
|
||
| # save cudagraph output buffer | ||
| if isinstance(outputs, torch.Tensor): | ||
| outputs = (outputs,) | ||
| self.fwd_graph_outputs = outputs | ||
| self.fwd_graph_output_surface = self.get_tensors(outputs) | ||
|
|
||
| if self.training and torch.is_grad_enabled(): | ||
| assert ( | ||
| len(self.fwd_graph_output_surface) > 0 | ||
| ), """Tried graphing a moudule that returned no tensors in training mode, | ||
| however the graphed module must output at least one tensor, | ||
| so that a corresponding backward node may be registered in the autograd graph.""" | ||
|
|
||
| # restore cached grads | ||
| for param in self.base_module.parameters(): | ||
| if hasattr(param, 'main_grad'): | ||
| saved_grad = save_main_grads.pop(0) | ||
| assert ( | ||
| param.main_grad.shape == saved_grad.shape | ||
| ), "Error restoring grads while cudagraphing!" | ||
| param.main_grad.copy_(saved_grad) | ||
|
|
||
| if self.fp8_enabled or self.fp4_enabled: | ||
| restore_fp8_tensors([self.base_module], saved_fp8_tensors) | ||
|
|
||
| # Unfreeze GC. | ||
| if FREEZE_GC: | ||
| gc.unfreeze() | ||
|
|
||
| # gc.collect() drops references to unreachable tensors created during capture, | ||
| # returning their storage to the allocator to avoid a slowdown during replay. However, | ||
| # it forces expensive global garbage collection, so must be done only on the last layer | ||
| # per-device to avoid slowing down graph creation. | ||
| if self.is_last_layer: | ||
| gc.collect() | ||
|
|
||
| def create_fwd_graph_old(self, args, kwargs, clone_inputs=True): | ||
| """Create a fwd cudagraph for this runner. Should be called inside | ||
| 'create_cudagraphs()'.""" | ||
|
|
||
| # Freeze GC, to speed up capture time ~15-20x. | ||
| if FREEZE_GC: | ||
| gc.freeze() | ||
|
|
||
| # save grads and other variables that may be affected by graph warmup | ||
| if self.training and torch.is_grad_enabled(): | ||
| save_main_grads = [ | ||
| param.main_grad.clone() | ||
| for param in self.base_module.parameters() | ||
| if hasattr(param, 'main_grad') | ||
| ] | ||
|
|
||
| saved_fp8_tensors = None | ||
|
|
||
| if self.fp8_enabled: | ||
| if is_te_min_version("1.13.0"): | ||
| saved_fp8_tensors = save_fp8_tensors([self.base_module], self.fp8_recipe) | ||
| else: | ||
| saved_fp8_tensors = save_fp8_tensors( | ||
| [self.base_module], self.fp8_recipe.amax_history_len | ||
| ) | ||
| elif self.fp4_enabled: | ||
| if is_te_min_version("2.7.0.dev0"): | ||
| saved_fp8_tensors = save_fp8_tensors([self.base_module], self.fp4_recipe) | ||
| else: | ||
| raise ValueError("FP4 requires TE >= 2.7.0.dev0 for NVFP4BlockScaling support.") | ||
|
|
||
| if clone_inputs: | ||
| args, kwargs = self.zero_out_tensors(args, kwargs) | ||
|
|
||
| input_tensors = self.get_tensors(args, kwargs) | ||
| self.fwd_graph_input_surface = input_tensors + tuple(self.base_module.parameters()) | ||
|
|
||
| self.fwd_graph = torch.cuda.CUDAGraph() | ||
|
|
||
| # For cases with multiple active RNG states, e.g. TP. | ||
| for _, state in get_all_rng_states().items(): | ||
| rng_states = get_all_rng_states() | ||
| with torch.inference_mode(mode=False): | ||
| for gen in rng_states.values(): | ||
| self.fwd_graph.register_generator_state( | ||
| _ensure_generator_state_is_cudagraph_safe(gen) | ||
| ) | ||
| self.fwd_graph.register_generator_state(state) | ||
|
|
||
| # warmup again as case graph capture mode may execute a different codepath | ||
| for _ in range(self.num_warmup_steps): | ||
| with self.get_quantization_context(): | ||
| outputs = self.base_module.forward(*args, **kwargs) | ||
| torch.cuda.synchronize() | ||
| # Register default CUDA generators ourselves (fixed in-place to have normal tensors) | ||
| # before capture begins, to avoid inference-tensor state issues during capture. | ||
| with torch.inference_mode(mode=False): | ||
| for device_idx in range(torch.cuda.device_count()): | ||
| default_gen = torch.cuda.default_generators[device_idx] | ||
| self.fwd_graph.register_generator_state( | ||
| _ensure_generator_state_is_cudagraph_safe(default_gen) | ||
| ) | ||
|
|
||
| with torch.cuda.graph( | ||
| self.fwd_graph, pool=self.fwd_mempool, capture_error_mode="thread_local" | ||
| ): | ||
| outputs = self.base_module.forward(*args, **kwargs) | ||
|
|
||
| if self.training and torch.is_grad_enabled(): | ||
| if isinstance(outputs, torch.Tensor): | ||
| outputs = (outputs,) | ||
|
|
@@ -704,6 +876,15 @@ def create_fwd_graph(self, args, kwargs, clone_inputs=True): | |
|
|
||
| with self.get_quantization_context(): | ||
| torch.cuda.synchronize() | ||
| # Register default CUDA generators ourselves (fixed in-place to have normal tensors) | ||
| # before capture begins, to avoid inference-tensor state issues during capture. | ||
| with torch.inference_mode(mode=False): | ||
| for device_idx in range(torch.cuda.device_count()): | ||
| default_gen = torch.cuda.default_generators[device_idx] | ||
| self.fwd_graph.register_generator_state( | ||
| _ensure_generator_state_is_cudagraph_safe(default_gen) | ||
| ) | ||
|
|
||
| with torch.cuda.graph( | ||
| self.fwd_graph, pool=self.fwd_mempool, capture_error_mode="thread_local" | ||
| ): | ||
|
|
@@ -1319,7 +1500,7 @@ def __call__(self, megatron_module, args, kwargs): | |
| # Now replay the graph | ||
| out = runner.replay_graph_capture(self.is_first_microbatch, args, kwargs) | ||
|
|
||
| elif self.training: | ||
| elif self.training and torch.is_grad_enabled(): | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not related specifically to this line or meant to be address in this MR, but do we ever need to be able to distinguish between no_grad and inference_mode in the code? If two forwards have the same signature but one is no grad and another is inference mode I wonder if it will error at all in inlace ops
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've made this unnecessary by flipping to
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. But yes you're right — right now we rely on cudagraph runner matching guardrails like correct shapes and arguments (e.g. inference passes some arguments which training does not). If we ever have graphs with the exact same shapes and arguments but differing only between inference mode and no_grad mode we may have issues. |
||
| # Training mode | ||
| runner = self.get_cudagraph_runner(megatron_module, args, kwargs) | ||
| # check if a layer is frozen during training. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Seems a bit dangerous to have Any and not throw an exception for unsupported data types. For instance, if I send a set of tensors, this will not clone them but return a set of tensors which a user of the function would expect to have cloned.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We actually don't want to throw an exception, but to pass-through for unsupported datatypes. This helper function is scoped very tightly within the cudagraphs code such that the "send set of tensors" case would *not be realistic, in my opinion.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I did not think of that! Got it now. The only q I have is why if I send a tuple/list of tensors we want to clone them, but if I send a set of tensors we don't clone them?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Sets of tensors are never used as cudagraph input/output containers — it wouldn't make sense because inputs are ordered, duplicates are meaningful, etc. I would be more concerned if we were sending in a set of tensors at all so I will just throw an error.
This function does see other types (e.g. generators) which need to pass through so I'm not going to modify the typing hints.