Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/gpu_unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ jobs:
uv run --frozen pytest -s -x tests/distributed/test_async_offload_unit.py
uv run --frozen pytest -s -x tests/distributed/test_dummy_forward.py
uv run --frozen pytest -s -x tests/distributed/test_reduce_scatter_fp32_accumulation.py
uv run --frozen pytest -s -x tests/distributed/test_gradient_checkpointing.py
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
uv run --frozen pytest -s -x tests/distributed/test_torch_compile.py
- name: Run data tests
run: |
Expand Down
5 changes: 3 additions & 2 deletions docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,9 @@ tests/
│ ├── test_balance_reverse.py # Balance/recovery precision (8 GPUs)
│ └── test_balance_sorting_algo.py # Post-MBS data sorting (CPU)
│
├── distributed/ # Multi-GPU training correctness
├── distributed/ # Distributed training and runtime correctness
│ ├── test_fsdp_equivalence.py # Single-GPU vs FSDP2 grad equivalence
│ ├── test_gradient_checkpointing.py # Checkpoint kwargs and recomputed-input grad cleanup
│ └── test_dummy_forward.py # Asymmetric multimodal forward (NCCL hang prevention)
│
├── e2e/ # End-to-end training integration
Expand Down Expand Up @@ -107,7 +108,7 @@ tests/
| **Ops / kernels** | `tests/ops/` | 0-1 GPU (SM90+ for Quack, DeepSeek-V4 TileLang, and mHC TileKernels) | pytest | Fused kernel guards, dispatch, correctness, and performance |
| **Data pipeline** | `tests/data/` | 0-1 GPU | pytest | Data loading, collation, preprocessing |
| **Parallelism** | `tests/parallel/` | 4-8 GPUs | torchrun / pytest | SP, EP, data-balance primitives |
| **FSDP correctness** | `tests/distributed/` | 2+ GPUs | torchrun (subprocess + mp.spawn) | Single-GPU vs FSDP2 equivalence, dummy forward |
| **Distributed runtime** | `tests/distributed/` | 0-2+ GPUs | pytest + torchrun | FSDP equivalence, dummy forward, gradient checkpointing |
| **E2E parallel** | `tests/e2e/` | 4+ GPUs | torchrun (subprocess) | SP/EP alignment across full training runs |
| **Checkpoints** | `tests/checkpoints/` | 0-8 GPUs | pytest + torchrun | Save/load, DCP→HF conversion |
| **Utilities** | `tests/utils/` | 0-8 GPUs | pytest + torchrun | FLOPs, grad clipping, weight broadcast |
Expand Down
30 changes: 30 additions & 0 deletions tests/distributed/test_gradient_checkpointing.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import types
import weakref

import pytest
import torch
import torch.nn as nn
from torch.utils.checkpoint import noop_context_fn

from veomni.arguments import GradientCheckpointingConfig, MixedPrecisionConfig
from veomni.distributed.checkpoint import CheckpointFunction
from veomni.distributed.torch_parallelize import build_parallelize_model


Expand All @@ -17,6 +20,20 @@ def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
self.gradient_checkpointing_kwargs = gradient_checkpointing_kwargs


class _RetainingCheckpointModule(nn.Module):
def __init__(self):
super().__init__()
self.recomputed_output = None
self.recomputed_input_ref = None

def forward(self, value):
output = value.sin()
if torch.is_grad_enabled():
self.recomputed_output = output
self.recomputed_input_ref = weakref.ref(value)
return output * 2


@pytest.mark.parametrize("early_stop", [True, False])
@pytest.mark.parametrize("use_reentrant", [True, False])
def test_build_parallelize_model_forwards_checkpoint_early_stop(monkeypatch, early_stop, use_reentrant):
Expand Down Expand Up @@ -49,3 +66,16 @@ def test_build_parallelize_model_forwards_checkpoint_early_stop(monkeypatch, ear

def test_gradient_checkpointing_config_enables_early_stop_by_default():
assert GradientCheckpointingConfig().early_stop is True


def test_reentrant_checkpoint_releases_recomputed_input_grad():
module = _RetainingCheckpointModule()
value = torch.randn(8, requires_grad=True)

output = CheckpointFunction.apply(module, False, value)
output.sum().backward()

recomputed_input = module.recomputed_input_ref()
assert recomputed_input is not None
assert recomputed_input.grad is None
torch.testing.assert_close(value.grad, 2 * value.detach().cos())
6 changes: 6 additions & 0 deletions veomni/distributed/checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,12 @@ def backward(ctx, *args):
torch.autograd.backward(outputs_with_grad, args_with_grad)
grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else None for inp in detached_inputs)

# The detached recomputation leaves can outlive this call through saved-variable references.
# Their gradients have been captured above, so release the stale references before returning.
for inp in detached_inputs:
if isinstance(inp, torch.Tensor):
inp.grad = None

# patch code, remove the extra allgather with use_reentrant + ckpt
if handle:
_post_backward_hook(state, handle, None)
Expand Down
Loading