diff --git a/megatron/core/optimizer/clip_grads.py b/megatron/core/optimizer/clip_grads.py index cb2f23a685f..1069d5ed3f2 100644 --- a/megatron/core/optimizer/clip_grads.py +++ b/megatron/core/optimizer/clip_grads.py @@ -204,7 +204,7 @@ def count_zeros_fp32( # - grad should not be none # - parameter should not be shared # - should not be a replica due to tensor model parallelism - total_num_zeros = torch.zeros(1, dtype=torch.float, device='cuda') + total_num_zeros = torch.zeros(1, dtype=torch.int64, device='cuda') data_parallel_group = None use_megatron_fsdp = False for param in parameters: