Skip to content
Closed
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
9d450b4
Add param group
Autumn1998 Apr 1, 2026
c76b84e
1. Add fsdp2/m-fsdp toy example for API align
shjwudp Mar 30, 2026
2fcea79
init fully_shard v2 FSDPModule and fsdp hooks
shjwudp Mar 30, 2026
463ae77
use post_backward do post_accumulate_grad job
shjwudp Mar 30, 2026
480c53b
fix import name
shjwudp Apr 1, 2026
79fb44e
fix set of bugs
shjwudp Apr 1, 2026
cafb667
fix FSDPModule param name mapping
shjwudp Apr 1, 2026
e4ebb50
Add get_state_dict support
shjwudp Apr 9, 2026
6b815e7
init fully_shard api mcore_fsdp_adapter support
shjwudp Apr 21, 2026
54eedd7
fix bug in mcore_fsdp_adapter + fully_shard_v2
shjwudp Apr 21, 2026
3ba54b9
add mcore fsdp + fully_shard_v2 ut
shjwudp Apr 21, 2026
2571a40
Add _materialize_meta_module function
shjwudp Apr 21, 2026
8e85301
fix chunk_size_factor & add mock param_and_grad_buffer
shjwudp Apr 22, 2026
66ede6e
Squash merge mfsdp_refactor_main_debug into mfsdp_refactor_main
shjwudp Apr 23, 2026
ca22fbe
refactor: reorganize fully_shard_v2 code to fully_shard_rewrite
shjwudp Apr 24, 2026
2278b25
fix: update fully_shard_rewrite imports and add docstrings
shjwudp Apr 24, 2026
06682e2
feat: add unshard prefetch and reduce-scatter overlap for fully_shard…
shjwudp Apr 26, 2026
950d304
fix: fix overlap bugs in fully_shard_rewrite and add design doc
shjwudp Apr 28, 2026
4844fe8
fix(mfsdp): convergence fix, memory optimization, and polish for over…
shjwudp May 6, 2026
58aadfb
feat(mfsdp): llama3-8b convergence fixes, debug tooling, and docs
shjwudp May 10, 2026
77c3ae0
Refactor fully shard rewrite after convergence fixes
Autumn1998 May 11, 2026
efbc8ac
Merge pull request #4 from Autumn1998/tongliu/fsdp-v2-mixed-precision
shjwudp May 11, 2026
2812aab
refactor(mfsdp): simplify mixed precision policy defaults
Autumn1998 May 11, 2026
3a896e2
Merge pull request #5 from Autumn1998/tongliu/fsdp-v2-mixed-precision
shjwudp May 11, 2026
8a2515b
Add FSDP v2 MXFP8 mixed precision support
Autumn1998 May 12, 2026
9d4ac5c
feat(mfsdp): activation recompute support, bucket allocator, and conv…
shjwudp May 13, 2026
7916ae0
rename fully_shard_rewrite
shjwudp May 13, 2026
6e504fc
polish(mfsdp): add copyright, fix lint warnings, and improve docs
shjwudp May 14, 2026
3a80853
Align FSDP v2 MXFP8 weight handling
Autumn1998 May 14, 2026
876a87b
Merge mfsdp refactor main into FSDP v2 MXFP8
Autumn1998 May 14, 2026
4e219fd
Clean up FSDP v2 bucket allocator ownership
Autumn1998 May 14, 2026
0c911b8
Merge pull request #6 from Autumn1998/tongliu/fsdp-v2-mixed-precision
shjwudp May 14, 2026
f775c03
Clean up FSDP v2 FP8 buffer binding
Autumn1998 May 15, 2026
b0da50c
fix(mfsdp): wgrad fusion support, allreduce propagation, and safety g…
shjwudp May 15, 2026
583505b
Move FSDP v2 storage decisions into policy
Autumn1998 May 15, 2026
ccc5662
Clean up FSDP v2 mixed precision binding
Autumn1998 May 15, 2026
fad3a42
Merge remote-tracking branch 'origin/mfsdp_refactor_main' into tongli…
Autumn1998 May 15, 2026
279c72e
test(fsdp): avoid duplicate grad accumulation fusion arg
Autumn1998 May 15, 2026
87b93df
Merge pull request #7 from Autumn1998/tongliu/fsdp-v2-mixed-precision
shjwudp May 18, 2026
9f90cb5
split: extract stage-2 MCore integration to mfsdp_refactor_main_stage2
shjwudp May 18, 2026
5d6c8a5
split: revert remaining MCore integration files to nvidia/main
shjwudp May 18, 2026
694c09d
revert ddp_config & megatron_fsdp.py
shjwudp May 18, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 249 additions & 0 deletions examples/megatron_fsdp/fsdp_toy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
import argparse
import os
import sys
from pathlib import Path
from typing import Tuple

import torch
import torch.nn as nn
import torch.distributed as dist

import torch.distributed.checkpoint as dcp
from torch.distributed.checkpoint.state_dict import get_state_dict, set_state_dict
from torch.distributed.checkpoint.stateful import Stateful

from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.tensor import DTensor


# -----------------------
# Model definitions
# -----------------------

class ToyBlock(nn.Module):
def __init__(self, dim: int):
super().__init__()
self.linear1 = nn.Linear(dim, dim)
self.linear2 = nn.Linear(dim, dim)

def forward(self, x):
return self.linear2(torch.relu(self.linear1(x)))


class ToyModel(nn.Module):
def __init__(self, dim: int, n_layers: int):
super().__init__()
self.layers = nn.ModuleList(ToyBlock(dim) for _ in range(n_layers))
self.out = nn.Linear(dim, dim)

def forward(self, x):
for layer in self.layers:
x = layer(x)
return self.out(x)


# -----------------------
# Distributed init / mesh
# -----------------------

def init_distributed() -> torch.distributed.device_mesh.DeviceMesh:
"""Initialize process group and device mesh."""
if not dist.is_initialized():
dist.init_process_group("nccl")
world_size = dist.get_world_size()
rank = dist.get_rank()
torch.cuda.set_device(rank)
mesh = init_device_mesh("cuda", mesh_shape=(world_size,))
return mesh


def build_fsdp_model(
dim: int,
n_layers: int,
use_megatron_fsdp: bool,
) -> Tuple["FSDPModule", torch.distributed.device_mesh.DeviceMesh]:
if use_megatron_fsdp:
from megatron.core.distributed.fsdp.src.megatron_fsdp.fully_shard_rewrite import fully_shard, FSDPModule
from megatron.core.distributed.fsdp.src.megatron_fsdp.uneven_dtensor import get_state_dict
sys.modules["get_state_dict"] = get_state_dict

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm pretty nervous about this monkeypatch. I sort of know your intention of keeping the optimizer unchanged but this arguably hides too much from the user.

else:
from torch.distributed.fsdp import fully_shard, FSDPModule

mesh = init_distributed()
model = ToyModel(dim=dim, n_layers=n_layers).to("cuda")

# Example: per-layer sharding
for layer in model.layers:
fully_shard(layer, mesh=mesh)

# Optionally shard the root as well
fully_shard(model, mesh=mesh)

assert isinstance(model, ToyModel)
assert isinstance(model, FSDPModule)

p = next(model.parameters())
assert isinstance(p, DTensor)
return model, mesh


# -----------------------
# Checkpoint helpers
# -----------------------

class AppState(Stateful):
"""This is a useful wrapper for checkpointing the Application State. Since this object is compliant
with the Stateful protocol, DCP will automatically call state_dict/load_stat_dict as needed in the
dcp.save/load APIs.

Note: We take advantage of this wrapper to hande calling distributed state dict methods on the model
and optimizer.
"""

def __init__(self, model, optimizer=None):
self.model = model
self.optimizer = optimizer

def state_dict(self):
# this line automatically manages FSDP FQN's, as well as sets the default state dict type to FSDP.SHARDED_STATE_DICT
model_state_dict, optimizer_state_dict = get_state_dict(self.model, self.optimizer)
return {
"model": model_state_dict,
"optim": optimizer_state_dict
}

def load_state_dict(self, state_dict):
# sets our state dicts on the model and optimizer, now that we've loaded
set_state_dict(
self.model,
self.optimizer,
model_state_dict=state_dict["model"],
optim_state_dict=state_dict["optim"]
)


def save_checkpoint(
model: nn.Module,
optimizer: torch.optim.Optimizer,
step: int,
ckpt_dir: str,
) -> None:
rank = dist.get_rank()
Path(ckpt_dir).mkdir(parents=True, exist_ok=True)

ckpt_step_dir = os.path.join(ckpt_dir, f"step_{step:06d}")

state = {"app": AppState(model, optimizer), "step": step}
dcp.save(state_dict=state, checkpoint_id=ckpt_step_dir)

if rank == 0:
print(f"[rank0] Saved checkpoint to {ckpt_dir}")


def load_checkpoint_if_available(
model: nn.Module,
optimizer: torch.optim.Optimizer,
ckpt_dir: str,
) -> int:
"""
Load the latest checkpoint if present.
Returns starting step (step+1) for training.
"""
if not os.path.exists(ckpt_dir):
return 0

all_ckpts = sorted(
[f for f in os.listdir(ckpt_dir) if f.startswith("step_")]
)
last_ckpt = os.path.join(ckpt_dir, all_ckpts[-1]) if all_ckpts else None

if last_ckpt is None:
return 0

step = torch.zeros([1])
state = {"app": AppState(model, optimizer), "step": step}
dcp.load(state_dict=state, checkpoint_id=last_ckpt)

return int(step.item()) + 1


# -----------------------
# Training loop
# -----------------------

def train(
args: argparse.Namespace,
model: nn.Module,
optimizer: torch.optim.Optimizer,
start_step: int = 0,
) -> None:
rank = dist.get_rank()
world_size = dist.get_world_size()

model.train()
step = start_step

for epoch in range(args.epochs):
for _ in range(args.steps_per_epoch):
# Dummy data
x = torch.randn(args.batch_size, args.model_dim, device="cuda")
y = model(x)
loss = y.sum() / (world_size * args.batch_size)
loss.backward()
optimizer.step()
optimizer.zero_grad(set_to_none=True)

if step % args.log_interval == 0 and rank == 0:
print(f"[rank0] epoch={epoch} step={step} loss={loss.item():.4f}")

if args.ckpt_dir and step % args.ckpt_interval == 0 and step > 0:
save_checkpoint(model, optimizer, step, args.ckpt_dir)

step += 1

# Final checkpoint
if args.ckpt_dir:
save_checkpoint(model, optimizer, step, args.ckpt_dir)


# -----------------------
# __main__ entry
# -----------------------

def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Toy FSDP2 training example")
parser.add_argument("--model-dim", type=int, default=1024)
parser.add_argument("--n-layers", type=int, default=3)
parser.add_argument("--batch-size", type=int, default=8)
parser.add_argument("--epochs", type=int, default=2)
parser.add_argument("--steps-per-epoch", type=int, default=10)
parser.add_argument("--lr", type=float, default=1e-3)
parser.add_argument("--ckpt-dir", type=str, default="checkpoints")
parser.add_argument("--ckpt-interval", type=int, default=20)
parser.add_argument("--log-interval", type=int, default=5)
parser.add_argument("--use-megatron-fsdp", action="store_true", help="Use Megatron-FSDP instead of PyTorch FSDP2")
return parser.parse_args()


def main() -> None:
args = parse_args()

model, _ = build_fsdp_model(
dim=args.model_dim,
n_layers=args.n_layers,
use_megatron_fsdp=args.use_megatron_fsdp,
)
optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr)

start_step = 0
if args.ckpt_dir:
start_step = load_checkpoint_if_available(model, optimizer, args.ckpt_dir)

train(args, model, optimizer, start_step=start_step)

dist.barrier()
dist.destroy_process_group()


if __name__ == "__main__":
main()
4 changes: 4 additions & 0 deletions megatron/core/distributed/distributed_data_parallel_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,10 @@ class DistributedDataParallelConfig:
main gradients to parameter dtype for `.grad`.
"""

use_fully_shard_api: bool = False
"""If true, use the `fully_shard` API for FSDP sharding the model.
"""

def __post_init__(self):
import os

Expand Down
Loading