Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion docs/models/deepseek/deepseek-v4-flash.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ DeepSeek V4 training tracking issue: [`radixark/miles#1046`](https://github.com/
| Model | Active / Total | HF ID |
|---|---|---|
| DeepSeek-V4-Flash | 13 B / 284 B | [sgl-project/DeepSeek-V4-Flash-FP8](https://huggingface.co/sgl-project/DeepSeek-V4-Flash-FP8) |
| DeepSeek-V4-Flash-0731 | 13 B / 284 B | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) |

`DeepSeek-V4-Flash-0731` is the official release with MXFP4 routed experts. Its architecture is identical
to DeepSeek-V4-Flash; the launcher's `prepare-fp8` stage (`tools/convert_mxfp4_to_fp8.py`) casts the
experts losslessly to blockwise FP8, after which the pipeline is identical to `DeepSeek-V4-Flash-FP8`.

## 3. Quick start

Expand All @@ -44,7 +49,15 @@ python scripts/run_deepseek_v4.py full-train \
--num-nodes 16 --num-gpus-per-node 8 --rollout-num-nodes 8
```

The `full-train` subcommand chains `prepare-download → prepare-single → prepare-spmd → prepare-cp → train`. Each stage has a sentinel-based skip so you can re-run safely after the first invocation.
```bash
# Official MXFP4 release (0731) — adds an MXFP4->FP8 cast before the same pipeline.
# 8-node GB300 run (32 GPUs):
python scripts/run_deepseek_v4.py full-train \
--model-name DeepSeek-V4-Flash-0731 \
--num-nodes 8 --num-gpus-per-node 4
```

The `full-train` subcommand chains `prepare-download → [prepare-fp8 →] prepare-single → prepare-spmd → prepare-cp → train` (`prepare-fp8` only for the MXFP4 `0731` variant). Each stage has a sentinel-based skip so you can re-run safely after the first invocation.

### 3.2 Launcher path defaults

Expand Down
2 changes: 1 addition & 1 deletion miles/backends/megatron_utils/rematerialize_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def _named_restore_extras(model: Sequence[torch.nn.Module]) -> Iterator[tuple[st
if "expert_bias" in name:
yield f"vp_stages.{vp_stage}.{strip_param_name_prefix(name)}", buffer
for name, param in model_module.named_parameters():
if param.dtype == torch.float32:
if param.dtype == torch.float32 or not param.requires_grad:
yield f"vp_stages.{vp_stage}.{strip_param_name_prefix(name)}", param


Expand Down
64 changes: 59 additions & 5 deletions scripts/run_deepseek_v4.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
smoke testing. **Cannot generate meaningful output -
pipeline-only sanity check.**
- DeepSeek-V4-Pro-FP8 Verified profile: 32 nodes x 8 GPUs on H200.
- DeepSeek-V4-Flash-0731 Official deepseek-ai release (MXFP4 routed experts).
prepare-fp8 casts the experts losslessly to blockwise
FP8; downstream is identical to DeepSeek-V4-Flash-FP8.

Usage patterns:

Expand All @@ -18,8 +21,9 @@
--model-name DeepSeek-V4-Flash-FP8-4layer \
--num-nodes 1 --num-gpus-per-node 8

2. Individual steps (download -> FP8->BF16 -> BF16->torch_dist -> rsync -> train):
2. Individual steps (download [-> MXFP4->FP8] -> FP8->BF16 -> BF16->torch_dist -> rsync -> train):
python scripts/run_deepseek_v4.py prepare-download --model-name DeepSeek-V4-Flash-FP8
python scripts/run_deepseek_v4.py prepare-fp8 --model-name DeepSeek-V4-Flash-0731
python scripts/run_deepseek_v4.py prepare-single --model-name DeepSeek-V4-Flash-FP8 \
--hf-checkpoint /root/models/DeepSeek-V4-Flash-FP8
python scripts/run_deepseek_v4.py prepare-spmd --model-name DeepSeek-V4-Flash-FP8 \
Expand All @@ -45,15 +49,20 @@
# 4-layer prune of sgl-project/DeepSeek-V4-Flash-FP8.
"DeepSeek-V4-Flash-FP8-4layer": "Pinaster",
"DeepSeek-V4-Pro-FP8": "sgl-project",
# Official release with MXFP4 routed experts; cast to FP8 by prepare-fp8.
"DeepSeek-V4-Flash-0731": "deepseek-ai",
}

_MEGATRON_MODEL_TYPE = {
"DeepSeek-V4-Flash-FP8": "deepseek-v4-flash",
"DeepSeek-V4-Flash-FP8-4layer": "deepseek-v4-flash-4layer",
"DeepSeek-V4-Pro-FP8": "deepseek-v4-pro",
"DeepSeek-V4-Flash-0731": "deepseek-v4-flash",
}

_PRO_MODEL_NAMES = ("DeepSeek-V4-Pro-FP8",)
_MXFP4_MODEL_NAMES = ("DeepSeek-V4-Flash-0731",)
_FLASH_FULL_MODEL_NAMES = ("DeepSeek-V4-Flash-FP8", "DeepSeek-V4-Flash-0731")
_BLACKWELL_HARDWARE = ("B200", "B300", "GB200", "GB300")

_DSV4_TE_PRECISION_CONFIG = """
Expand All @@ -79,6 +88,7 @@ class ScriptArgs(U.ExecuteTrainConfig):
"DeepSeek-V4-Flash-FP8",
"DeepSeek-V4-Flash-FP8-4layer",
"DeepSeek-V4-Pro-FP8",
"DeepSeek-V4-Flash-0731",
] = "DeepSeek-V4-Flash-FP8"

task: Literal["dapo_aime", "gsm8k"] = "dapo_aime"
Expand Down Expand Up @@ -152,6 +162,12 @@ def __post_init__(self):
def megatron_model_type(self):
return _MEGATRON_MODEL_TYPE[self.model_name]

@property
def fp8_name(self):
if self.model_name in _MXFP4_MODEL_NAMES:
return f"{self.model_name}-FP8"
return self.model_name

@property
def torch_dist_name(self):
return f"{self.model_name}_torch_dist"
Expand All @@ -171,7 +187,7 @@ def rollout_name(self):
if self.rollout_mxfp8:
return self.mxfp8_name
if self.rollout_fp8:
return self.model_name
return self.fp8_name
return self.bf16_name


Expand Down Expand Up @@ -234,10 +250,32 @@ def prepare_download(args: ScriptArgs):
_prepare_download(args)


def _prepare_fp8(args: ScriptArgs):
"""MXFP4 experts -> blockwise FP8 (lossless cast). Only for official MXFP4 releases."""
if args.model_name not in _MXFP4_MODEL_NAMES:
print(f"[prepare_fp8] {args.model_name} has no MXFP4 source; nothing to do.")
return
U.exec_command_gpu(
f"python tools/convert_mxfp4_to_fp8.py "
f"--model-dir {_hf_checkpoint_path(args)} "
f"--save-dir {args.model_dir}/{args.fp8_name} "
)


@app.command()
@U.dataclass_cli
def prepare_fp8(args: ScriptArgs):
"""MXFP4 -> FP8 cast (needs prepare-download done first). One node."""
_prepare_fp8(args)


def _prepare_single(args: ScriptArgs):
_download_dataset(args)

src = _hf_checkpoint_path(args)
if args.model_name in _MXFP4_MODEL_NAMES:
src = f"{args.model_dir}/{args.fp8_name}"
else:
src = _hf_checkpoint_path(args)
U.fp8_cast_bf16(
path_src=src,
path_dst=f"{args.model_dir}/{args.bf16_name}/",
Expand Down Expand Up @@ -283,7 +321,7 @@ def _prepare_spmd(args: ScriptArgs):
extra_args += (
"--tensor-model-parallel-size 1 " "--pipeline-model-parallel-size 1 " "--expert-model-parallel-size 1 "
)
elif actor_num_nodes == 8 and args.model_name == "DeepSeek-V4-Flash-FP8":
elif actor_num_nodes == 8 and args.model_name in _FLASH_FULL_MODEL_NAMES:
extra_args += (
"--tensor-model-parallel-size 1 "
"--pipeline-model-parallel-size 8 "
Expand Down Expand Up @@ -380,6 +418,7 @@ def _get_parallel_config(args: ScriptArgs) -> str:
"--decoder-first-pipeline-num-layers 4 "
"--decoder-last-pipeline-num-layers 3 "
"--context-parallel-size 2 "
"--allgather-cp "
"--expert-model-parallel-size 4 "
"--expert-tensor-parallel-size 1 "
)
Expand Down Expand Up @@ -418,7 +457,7 @@ def _get_parallel_config(args: ScriptArgs) -> str:
def _train(args: ScriptArgs):
if args.train_mxfp8 or args.rollout_mxfp8:
assert _is_blackwell(args), "MXFP8 requires Blackwell (B200/B300/GB200/GB300)"
if not args.rollout_fp8 or args.hf_checkpoint is None:
if not args.rollout_fp8 or args.hf_checkpoint is None or args.model_name in _MXFP4_MODEL_NAMES:
rollout_checkpoint = f"{args.model_local_dir}/{args.rollout_name}"
if args.hf_checkpoint != rollout_checkpoint:
print(f"[precision] rollout checkpoint: {args.hf_checkpoint} -> {rollout_checkpoint}")
Expand Down Expand Up @@ -526,6 +565,12 @@ def _train(args: ScriptArgs):
sglang_tp_size = 32
sglang_dp_size = 32
sglang_ep_size = 32
elif args.num_gpus_per_node == 4:
# GB300, use tp=8. tp=4 causes CPU OOM when colocate
sglang_world_size = 8
sglang_tp_size = 8
sglang_dp_size = 1
sglang_ep_size = 8
else:
sglang_world_size = 4
sglang_tp_size = 4
Expand Down Expand Up @@ -574,6 +619,8 @@ def _train(args: ScriptArgs):
"SGLANG_HEALTH_CHECK_TIMEOUT": "120",
"SGLANG_DG_CACHE_DIR_PER_PROCESS": "1",
"SGLANG_OPT_FP8_WO_A_GEMM": "0",
# Colocated multi-engine init can deadlock in the multimem all-gather rendezvous (sgl-project/sglang#36110).
"SGLANG_DISABLE_MULTIMEM_AG": "1",
}
if args.model_name == "DeepSeek-V4-Pro-FP8":
extra_env_vars["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "256"
Expand Down Expand Up @@ -678,6 +725,13 @@ def train(args: ScriptArgs):
def full_train(args: ScriptArgs):
_prepare_download(args)

if args.model_name in _MXFP4_MODEL_NAMES:
fp8_sentinel = Path(f"{args.model_dir}/{args.fp8_name}") / "model.safetensors.index.json"
if not fp8_sentinel.exists():
_prepare_fp8(args)
else:
print(f"[full_train] Skipping MXFP4->FP8 cast: {fp8_sentinel} already exists.")

bf16_dir = Path(f"{args.model_dir}/{args.bf16_name}")
bf16_sentinel = bf16_dir / "model.safetensors.index.json"
if not bf16_sentinel.exists():
Expand Down
5 changes: 4 additions & 1 deletion tests/manual/launch_scripts/test_py_launch_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,10 @@ def _nemotron_checkpoint(sandbox: Path) -> dict[str, object]:
},
}

_ENTRYPOINTS_DISABLED_BY_THEIR_OWN_DEFAULTS = {("scripts/run_deepseek_v4.py", "prepare_mxfp8")}
_ENTRYPOINTS_DISABLED_BY_THEIR_OWN_DEFAULTS = {
("scripts/run_deepseek_v4.py", "prepare_mxfp8"),
("scripts/run_deepseek_v4.py", "prepare_fp8"),
}

_SCRIPTS = [
script for script in iter_py_launch_scripts() if script.rel not in _SCRIPTS_IMPORTABLE_ONLY_UNDER_THE_NPU_PATCH
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l
### 11
export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit
--address="http://127.0.0.1:8265"
--runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "SGLANG_SKIP_CHECKPOINT_LOAD_CHECK": "1", "SGLANG_DSV4_FP4_EXPERTS": "0", "SGLANG_HEALTH_CHECK_TIMEOUT": "120", "SGLANG_DG_CACHE_DIR_PER_PROCESS": "1", "SGLANG_OPT_FP8_WO_A_GEMM": "0", "NCCL_ALGO": "Ring", "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0", "CUBLAS_WORKSPACE_CONFIG": ":4096:8", "PYTHONPATH": "<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath"}}'
--runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "SGLANG_SKIP_CHECKPOINT_LOAD_CHECK": "1", "SGLANG_DSV4_FP4_EXPERTS": "0", "SGLANG_HEALTH_CHECK_TIMEOUT": "120", "SGLANG_DG_CACHE_DIR_PER_PROCESS": "1", "SGLANG_OPT_FP8_WO_A_GEMM": "0", "SGLANG_DISABLE_MULTIMEM_AG": "1", "NCCL_ALGO": "Ring", "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0", "CUBLAS_WORKSPACE_CONFIG": ":4096:8", "PYTHONPATH": "<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath"}}'
-- python3 <REPO_ROOT>/train.py
--disable-bias-linear
--num-layers 4
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l
### 3
export no_proxy=127.0.0.1 && export PYTHONUNBUFFERED=1 && ray job submit
--address="http://127.0.0.1:8265"
--runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "SGLANG_SKIP_CHECKPOINT_LOAD_CHECK": "1", "SGLANG_DSV4_FP4_EXPERTS": "0", "SGLANG_HEALTH_CHECK_TIMEOUT": "120", "SGLANG_DG_CACHE_DIR_PER_PROCESS": "1", "SGLANG_OPT_FP8_WO_A_GEMM": "0", "NCCL_ALGO": "Ring", "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0", "CUBLAS_WORKSPACE_CONFIG": ":4096:8", "PYTHONPATH": "<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath"}}'
--runtime-env-json='{"env_vars": {"PYTHONUNBUFFERED": "1", "CUDA_DEVICE_MAX_CONNECTIONS": "1", "NCCL_NVLS_ENABLE": "0", "no_proxy": "127.0.0.1,127.0.0.1", "MASTER_ADDR": "127.0.0.1", "SGLANG_SKIP_CHECKPOINT_LOAD_CHECK": "1", "SGLANG_DSV4_FP4_EXPERTS": "0", "SGLANG_HEALTH_CHECK_TIMEOUT": "120", "SGLANG_DG_CACHE_DIR_PER_PROCESS": "1", "SGLANG_OPT_FP8_WO_A_GEMM": "0", "SGLANG_DISABLE_MULTIMEM_AG": "1", "NCCL_ALGO": "Ring", "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0", "CUBLAS_WORKSPACE_CONFIG": ":4096:8", "PYTHONPATH": "<REPO_ROOT>:/root/Megatron-LM:/frozen/pythonpath"}}'
-- python3 <REPO_ROOT>/train.py
--disable-bias-linear
--num-layers 4
Expand Down
108 changes: 108 additions & 0 deletions tools/convert_mxfp4_to_fp8.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""
Convert an official DeepSeek-V4 checkpoint with MXFP4 routed experts to uniform
blockwise-FP8 (the sgl-project/DeepSeek-V4-*-FP8 layout).

Packed-e2m1fn expert weights (int8, with per-(1,32)-block ue8m0 scales) are cast
losslessly to e4m3fn with (128,128)-block e8m0 scales; all other tensors are
copied unchanged. `expert_dtype` is dropped from config.json.

python tools/convert_mxfp4_to_fp8.py --model-dir <mxfp4-hf> --save-dir <fp8-hf>
"""

import json
import os
from argparse import ArgumentParser
from pathlib import Path

import torch
from safetensors import safe_open
from safetensors.torch import load_file, save_file
from tqdm import tqdm

FP4_TABLE = torch.tensor(
[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
dtype=torch.float32,
)


# copied from deepseek-ai/DeepSeek-V4-Flash-0731 inference/convert.py
def cast_e2m1fn_to_e4m3fn(x: torch.Tensor, scale: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
assert x.dtype == torch.int8 and x.ndim == 2
out_dim, in_dim = x.size(0), x.size(1) * 2
fp8_block_size, fp4_block_size = 128, 32
assert out_dim % fp8_block_size == 0 and in_dim % fp8_block_size == 0
assert scale.shape == (out_dim, in_dim // fp4_block_size), f"{scale.shape=} {x.shape=}"

table = FP4_TABLE.to(x.device)
x = x.view(torch.uint8)
x = torch.stack([table[(x & 0x0F).long()], table[(x >> 4).long()]], dim=-1).reshape(out_dim, in_dim)

max_offset = 2**6
b_out, b_in = out_dim // fp8_block_size, in_dim // fp8_block_size
x = x.view(b_out, fp8_block_size, b_in, fp8_block_size).transpose(1, 2)
scale = scale.float().view(b_out, fp8_block_size, b_in, -1).transpose(1, 2).flatten(2)
tile_scale = scale.amax(dim=-1, keepdim=True) / max_offset
offset = (scale / tile_scale).unflatten(-1, (fp8_block_size, -1)).repeat_interleave(fp4_block_size, dim=-1)
x = (x * offset).transpose(1, 2).reshape(out_dim, in_dim)
# ue8m0 semantics (power-of-two values) stored as float32, matching the
# sgl-project FP8 repackages and the float32 scales on attention/dense.
return x.to(torch.float8_e4m3fn), tile_scale.squeeze(-1).float()


def main(model_dir: str, save_dir: str, device: str):
os.makedirs(save_dir, exist_ok=True)

config = json.loads((Path(model_dir) / "config.json").read_text())
config.pop("expert_dtype", None)
if isinstance(config.get("quantization_config"), dict):
config["quantization_config"].pop("expert_dtype", None)
(Path(save_dir) / "config.json").write_text(json.dumps(config, indent=2) + "\n")
for pattern in ("tokenizer*", "chat_template*", "generation_config.json"):
os.system(f"cp -rf {model_dir}/{pattern} {save_dir}/ 2>/dev/null")

index = json.loads((Path(model_dir) / "model.safetensors.index.json").read_text())
weight_map = dict(index["weight_map"])
shard_names = sorted(set(weight_map.values()))

fp4_weights = set()
for shard_name in shard_names:
with safe_open(f"{model_dir}/{shard_name}", framework="pt") as f:
for name in f.keys():
if f.get_slice(name).get_dtype() == "I8":
assert name.endswith(".weight"), f"unexpected int8 tensor {name}"
fp4_weights.add(name)
fp4_scales = {name.removesuffix(".weight") + ".scale" for name in fp4_weights}

for shard_name in tqdm(shard_names):
state_dict = load_file(f"{model_dir}/{shard_name}", device=device)
new_state_dict = {}
for name, tensor in state_dict.items():
if name in fp4_weights:
scale_name = name.removesuffix(".weight") + ".scale"
scale = state_dict.get(scale_name)
if scale is None:
scale = load_file(f"{model_dir}/{weight_map[scale_name]}", device=device)[scale_name]
weight, tile_scale = cast_e2m1fn_to_e4m3fn(tensor, scale)
new_state_dict[name] = weight
new_state_dict[scale_name] = tile_scale
weight_map[scale_name] = shard_name
elif name in fp4_scales:
continue
else:
new_state_dict[name] = tensor
save_file(new_state_dict, f"{save_dir}/{shard_name}")

(Path(save_dir) / "model.safetensors.index.json").write_text(
json.dumps({"metadata": {}, "weight_map": weight_map}, indent=2) + "\n"
)
assert fp4_weights, "no packed-fp4 expert weights found; is this already an FP8 checkpoint?"
print(f"cast {len(fp4_weights)} expert weights to fp8, saved to {save_dir}")


if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--model-dir", type=str, required=True)
parser.add_argument("--save-dir", type=str, required=True)
parser.add_argument("--device", type=str, default="cuda")
args = parser.parse_args()
main(args.model_dir, args.save_dir, args.device)
5 changes: 3 additions & 2 deletions tools/fp8_cast_bf16.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,11 +97,12 @@ def get_tensor(tensor_name):
try:
# Get scale_inv from the correct file
scale_inv = get_tensor(scale_inv_name)
fp8_weight_names.append(weight_name)
new_state_dict[weight_name] = weight_dequant(weight, scale_inv)
except KeyError:
print(f"Warning: Missing scale_inv tensor for {weight_name}, skipping conversion")
new_state_dict[weight_name] = weight
else:
fp8_weight_names.append(weight_name)
new_state_dict[weight_name] = weight_dequant(weight, scale_inv.float())
else:
new_state_dict[weight_name] = weight

Expand Down
Loading