Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions megatron/core/models/common/language_module/language_module.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,17 @@ def check_and_set_env_variable(
check_and_set_env_variable("NVTE_FUSED_ATTN", 1, AttnBackend.auto)
check_and_set_env_variable("NVTE_UNFUSED_ATTN", 1, AttnBackend.auto)

# Pin the FlashAttention generation for TransformerEngine by disabling the
# other versions via NVTE_FLASH_ATTN_V2/V3/V4 (default 1). This keeps the
# training-side attention on the same kernel as the mcore inference path,
# which honors config.flash_attention_version directly.
if self.config.flash_attention_version is not None:
for version in (2, 3, 4):
if version != self.config.flash_attention_version:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it make sense to have the else case explicitly set version = 1? So

else:
     check_and_set_env_variable(
         f"NVTE_FLASH_ATTN_V{version}", 1, self.config.attention_backend
     )

check_and_set_env_variable(
f"NVTE_FLASH_ATTN_V{version}", 0, self.config.attention_backend
)

def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor:
"""Computes the language model loss (Cross entropy across vocabulary)

Expand Down
36 changes: 31 additions & 5 deletions megatron/core/transformer/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,7 @@ def __init__(
self.attn_mask_type = attn_mask_type
self.attention_type = attention_type
self.batch_invariant_mode = config.batch_invariant_mode
self.flash_attention_version = config.flash_attention_version

# Cache the YaRN concentration factor (a.k.a. attention factor / mscale),
# which is a pure function of the config and is reused on every forward
Expand Down Expand Up @@ -983,6 +984,29 @@ def _flash_attention_3_forward_wrapper(
)
return output_total, softmax_lse

def _resolve_flash_version(self) -> Tuple[bool, bool]:
"""Resolve which FlashAttention generation this attention should run.

Honors ``config.flash_attention_version`` when pinned, otherwise falls back
to the auto preference order (FA4 > FA3 > FA2). Returns ``(use_fa4, use_fa3)``;
when both are False the FA2 kernel is used.
"""
pinned = self.flash_attention_version
if pinned == 4:
assert (
HAVE_FA4
), "flash_attention_version=4 requested but FlashAttention-4 is not installed"
return True, False
if pinned == 3:
assert (
HAVE_FA3
), "flash_attention_version=3 requested but FlashAttention-3 is not installed"
return False, True
if pinned == 2:
return False, False
# Auto: prefer the newest available generation.
return HAVE_FA4, (HAVE_FA3 and not HAVE_FA4)

def flash_decode_and_prefill(
self,
q: Tensor,
Expand Down Expand Up @@ -1041,14 +1065,16 @@ def flash_decode_and_prefill(
# the sink (off-by-one / learnable) softmax correction post-hoc.
need_lse = softmax_offset is not None

use_fa4, use_fa3 = self._resolve_flash_version()

# Flash attn kernel.
if not is_decode_only:
q = q.squeeze(1)
if getattr(self, "softmax_scale", None) is not None:
softmax_scale = self.softmax_scale
else:
softmax_scale = q.shape[-1] ** -0.5
if HAVE_FA4:
if use_fa4:
output_total, softmax_lse = flash_attn4_varlen_func(
q,
k,
Expand All @@ -1063,7 +1089,7 @@ def flash_decode_and_prefill(
window_size=window_size,
num_splits=1,
)
elif HAVE_FA3:
elif use_fa3:
# TODO(ksanthanam): Replace with call to flash_attn_varlen_func once
# it accepts block_table
fa3_ret = self._flash_attention_3_forward_wrapper(
Expand Down Expand Up @@ -1164,7 +1190,7 @@ def flash_decode_and_prefill(
output_total, softmax_lse, softmax_offset
)
else:
if HAVE_FA4:
if use_fa4:
if getattr(self, "softmax_scale", None) is not None:
softmax_scale = self.softmax_scale
else:
Expand Down Expand Up @@ -1207,12 +1233,12 @@ def flash_decode_and_prefill(
"softmax_scale": softmax_scale,
"causal": True,
"window_size": window_size,
"page_table" if HAVE_FA3 else "block_table": block_table,
"page_table" if use_fa3 else "block_table": block_table,
"num_splits": 0 if not self.batch_invariant_mode else 1,
}
if need_lse:
flash_attn_args["return_softmax_lse"] = True
if HAVE_FA3:
if use_fa3:
kvcache_ret = flash_attn3_with_kvcache(**flash_attn_args)
else:
assert (
Expand Down
109 changes: 108 additions & 1 deletion megatron/core/transformer/custom_layers/batch_invariant_kernels.py
Original file line number Diff line number Diff line change
Expand Up @@ -525,10 +525,15 @@ def get_batch_invariant_attention_block_size() -> AttentionBlockSize:
_MEG_TE_GENERAL_GEMM_ORIG = None
_TE_RMSNORM_FUNC_ORIGS: Dict[str, Any] = {}
_TE_GEMM_FUNC_ORIGS: Dict[str, Any] = {}
_TE_APPLY_NORM_ORIGS: Dict[str, Any] = {}


def _import_module_if_available(name: str):
spec = importlib.util.find_spec(name)
try:
spec = importlib.util.find_spec(name)
except ModuleNotFoundError:
# find_spec on a submodule raises when the parent package is absent.
return None
if spec is None:
return None
return importlib.import_module(name)
Expand Down Expand Up @@ -617,6 +622,34 @@ def _patched(*args, **kwargs):
_TE_RMSNORM_FUNC_ORIGS[name] = orig
setattr(te_layernorm_mod, name, _make_rmsnorm_patched(orig))

# Patch the fused-module normalization entry (`apply_normalization`). TE's
# fused LayerNormLinear / LayerNormMLP call this instead of RMSNorm.forward,
# so without this patch their internal RMSNorm runs TE's tex kernel, whose
# within-row reduction strategy depends on the total row count — i.e. it is
# NOT batch-invariant (observed: same rows, different output at 928 vs 2274
# rows on GB200, 1 bf16 ulp per layer, amplifying across depth).
import transformer_engine.pytorch.module._common as te_common

for mod_name, mod in (
("module._common", te_common),
(
"module.layernorm_linear",
_import_module_if_available("transformer_engine.pytorch.module.layernorm_linear"),
),
(
"module.layernorm_mlp",
_import_module_if_available("transformer_engine.pytorch.module.layernorm_mlp"),
),
):
key = f"{mod_name}.apply_normalization"
if (
mod is not None
and hasattr(mod, "apply_normalization")
and key not in _TE_APPLY_NORM_ORIGS
):
_TE_APPLY_NORM_ORIGS[key] = mod.apply_normalization
mod.apply_normalization = _te_apply_normalization_patched


def _te_unpatch_for_batch_invariant():
"""Restore original Transformer Engine functions if they were patched."""
Expand Down Expand Up @@ -652,6 +685,14 @@ def _te_unpatch_for_batch_invariant():
elif meg_te is None:
_MEG_TE_GENERAL_GEMM_ORIG = None

# Restore fused-module apply_normalization entries
for key, orig in list(_TE_APPLY_NORM_ORIGS.items()):
mod_name = key.rsplit(".apply_normalization", 1)[0]
mod = _import_module_if_available(f"transformer_engine.pytorch.{mod_name}")
if mod is not None and hasattr(mod, "apply_normalization"):
mod.apply_normalization = orig
_TE_APPLY_NORM_ORIGS.pop(key, None)

# Restore TE module-level RMSNorm functions
te_layernorm_mod = _import_module_if_available("transformer_engine.pytorch.module.layernorm")
if te_layernorm_mod is not None:
Expand Down Expand Up @@ -827,6 +868,72 @@ def backward(ctx, grad_output: torch.Tensor):
return dA, dB, dbias, None, None


def _te_apply_normalization_patched(
inputmat,
ln_out,
ln_weight,
ln_bias,
eps,
output_quantizer,
output_dtype,
normalization,
fwd_ln_sm_margin,
zero_centered_gamma,
):
"""Batch-invariant replacement for TE's fused-module `apply_normalization`.

Routes RMSNorm through the batch-invariant implementation (fp32 stats via
`mean_dim`, deterministic within-row reduction independent of row count).
Falls back to the original TE kernel for configurations the BI path does
not cover (LayerNorm, fp8 quantized output).

Returns `(ln_out, mu, rsigma)` like TE: `mu` is None for RMSNorm and
`rsigma` is fp32 with shape [rows], which TE's rmsnorm backward consumes.
"""
orig = _TE_APPLY_NORM_ORIGS.get("module._common.apply_normalization")
if (
not is_batch_invariant_mode_enabled()
or normalization != "RMSNorm"
or output_quantizer is not None
or ln_bias is not None
):
assert orig is not None, "TE apply_normalization original not captured"
return orig(
inputmat,
ln_out,
ln_weight,
ln_bias,
eps,
output_quantizer,
output_dtype,
normalization,
fwd_ln_sm_margin,
zero_centered_gamma,
)

x_fp32 = inputmat.float()
w_fp32 = ln_weight.float()
if zero_centered_gamma:
w_fp32 = w_fp32 + 1.0
ms = mean_dim(x_fp32 * x_fp32, dim=-1, keepdim=True)
rsigma = torch.rsqrt(ms + eps)
out_fp32 = (x_fp32 * rsigma) * w_fp32

# The fused-module callers (layernorm_linear / layernorm_mlp) pass a torch.dtype
# output_dtype (inputmat.dtype); assert that so we never silently ignore a TE
# DType or other unexpected value here.
assert isinstance(output_dtype, torch.dtype), (
"batch-invariant apply_normalization expects a torch.dtype output_dtype, got "
f"{type(output_dtype)}"
)
if ln_out is not None:
# copy_ casts to ln_out.dtype in place, avoiding an intermediate allocation.
ln_out.copy_(out_fp32)
else:
ln_out = out_fp32.to(output_dtype)
return ln_out, None, rsigma.squeeze(-1)


def _te_general_gemm_patched(*args, **kwargs) -> List[torch.Tensor]:
"""
Batch-invariant replacement for TE general_gemm.
Expand Down
35 changes: 34 additions & 1 deletion megatron/core/transformer/transformer_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,16 @@ class TransformerConfig(ModelParallelConfig):
If attention backend is local we use the local pytorch implementation in mcore.
Users can specify exact backend by changing this config. """

flash_attention_version: Optional[Literal[2, 3, 4]] = None
"""Pin the FlashAttention generation (2, 3, or 4) used by both the training
(TransformerEngine) and inference (mcore dynamic-batching) attention paths. When
None, each path selects a version automatically based on what is installed. Pinning
is required for batch-invariant mode: the training-side logprob recompute and the
inference engine must run the same kernel, since different FlashAttention
generations use different tile sizes and softmax accumulation orders and therefore
differ bitwise. On the training side this is enforced via TransformerEngine's
NVTE_FLASH_ATTN_V2/V3/V4 selection environment variables."""

softmax_scale: Optional[float] = None
"""Softmax scale for attention scaling."""

Expand Down Expand Up @@ -2807,10 +2817,33 @@ def _scope_to_str(s):
"for inference_optimized transformer implementation."
)

if self.flash_attention_version is not None:
assert self.flash_attention_version in (2, 3, 4), (
"flash_attention_version must be one of 2, 3, or 4, got "
f"{self.flash_attention_version}"
)

if self.batch_invariant_mode:
assert (
self.attention_backend == AttnBackend.flash
), "Batch invariant mode only supports FlashAttention"
), "Batch invariant mode only supports FlashAttention (--attention-backend flash)"
# The training (TransformerEngine) and inference attention paths must run
# the same FlashAttention kernel, so the version cannot be left to each
# path's autodetection. FlashAttention-2 is excluded because it does not
# expose the fixed num_splits schedule the batch-invariant kernels require.
assert self.flash_attention_version in (3, 4), (
"Batch invariant mode requires --flash-attention-version 3 or 4 so the "
"training and inference attention paths run the same batch-invariant "
f"FlashAttention kernel (got {self.flash_attention_version})."
)
# Context parallelism routes through TE's FA2 fwd/bwd kernels directly, which
# cannot be pinned to another version; dropout is not batch-invariant.
assert (
self.context_parallel_size == 1
), "Batch invariant mode does not support context parallelism"
assert (
self.attention_dropout == 0.0
), "Batch invariant mode does not support attention dropout"


@dataclass
Expand Down
14 changes: 9 additions & 5 deletions megatron/rl/rl_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1688,11 +1688,13 @@ def prepare_data_for_update(
if inference_logprobs is not None:
# Pack the inference logprobs using the helper function
# We do this for logging purposes even if is_correction is disabled
packed_inference_logprobs = pack_inference_logprobs(
inference_logprobs=packing_context.original_inference_logprobs,
packing_info=packing_context.packing_info,
generation_masks=packing_context.original_generation_masks,
bin_size=args.seq_length,
packed_inference_logprobs, packed_inference_filled_mask = (
pack_inference_logprobs(
inference_logprobs=packing_context.original_inference_logprobs,
packing_info=packing_context.packing_info,
generation_masks=packing_context.original_generation_masks,
bin_size=args.seq_length,
)
)

# Compute statistics for logging using packed data
Expand All @@ -1701,8 +1703,10 @@ def prepare_data_for_update(
packed_inference_logprobs=packed_inference_logprobs,
packed_loss_mask=packing_context.packed_loss_mask,
group_stats=group_stats,
filled_mask=packed_inference_filled_mask,
)


# Store packed inference logprobs in packing context
packing_context.packed_inference_logprobs = packed_inference_logprobs.cuda()
# Only mark as having inference logprobs for IS correction if enabled
Expand Down
Loading
Loading