Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion megatron/core/distributed/distributed_data_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,9 @@ def hook(*unused):

if param in self.param_to_bucket_group:
assert param.requires_grad
if self.ddp_config.overlap_grad_reduce:
if self.ddp_config.overlap_grad_reduce and not getattr(
param, 'is_cudagraph_output', False
):
assert (
param.grad is not None
), 'param.grad being None is not safe when overlap_grad_reduce is True'
Expand Down
Loading