diff --git a/megatron/core/distributed/distributed_data_parallel.py b/megatron/core/distributed/distributed_data_parallel.py
index 06f3fb32238..a711f1405d1 100644
--- a/megatron/core/distributed/distributed_data_parallel.py
+++ b/megatron/core/distributed/distributed_data_parallel.py
@@ -265,10 +265,18 @@ def __init__(
         # If bucketing is explicitly disabled, then put all buckets in a buffer into a single
         # bucket group.
         self.bucket_groups = partition_buckets(
-            self.buffers, force_single_bucket_group=disable_bucketing
+            self.buffers,
+            force_single_bucket_group=disable_bucketing,
+            reduce_scatter_with_fp32_accumulation=(
+                self.ddp_config.reduce_scatter_with_fp32_accumulation
+            ),
         )
         self.expert_parallel_bucket_groups = partition_buckets(
-            self.expert_parallel_buffers, force_single_bucket_group=disable_bucketing
+            self.expert_parallel_buffers,
+            force_single_bucket_group=disable_bucketing,
+            reduce_scatter_with_fp32_accumulation=(
+                self.ddp_config.reduce_scatter_with_fp32_accumulation
+            ),
         )
 
         if self.ddp_config.num_distributed_optimizer_instances > 1:
diff --git a/megatron/core/fp4_utils.py b/megatron/core/fp4_utils.py
index 245a04eb39e..45e57285a8d 100644
--- a/megatron/core/fp4_utils.py
+++ b/megatron/core/fp4_utils.py
@@ -81,7 +81,8 @@ def modify_nvfp4_rowwise_storage(fp4_tensor: torch.Tensor, new_rowwise_data: tor
     ), "Rowwise NVFP4 storage must be uint8"
     # Preserve existing values and then swap storage
     new_rowwise_data.detach().copy_(old_rowwise)
-    setattr(fp4_tensor, "_rowwise_data", new_rowwise_data)
+    fp4_tensor._rowwise_data = new_rowwise_data
+    del old_rowwise
 
 
 def quantize_nvfp4_param_shard(