Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions miles/backends/megatron_utils/actor.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,14 +112,14 @@ def init(

from megatron.core import mpu

if mpu.get_context_parallel_world_size() > 1:
from miles.backends.training_utils.cp_utils import setup_hybrid_cp
cp_world_size = mpu.get_context_parallel_world_size()
if cp_world_size > 1:
from miles.backends.training_utils.cp_utils import detect_and_setup_hybrid_cp

cp_group = mpu.get_context_parallel_group()
cp_rank = mpu.get_context_parallel_rank()
cp_world_size = mpu.get_context_parallel_world_size()
for model_chunk in self.model:
setup_hybrid_cp(model_chunk, cp_group, cp_rank, cp_world_size)
detect_and_setup_hybrid_cp(model_chunk, cp_group, cp_rank, cp_world_size)

verify_megatron_parallel_state(self.model)

Expand Down
42 changes: 35 additions & 7 deletions miles/backends/training_utils/cp_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@

from .parallel import get_parallel_state

try:
from fla.ops.cp import build_cp_context as _fla_build_cp_context
except ImportError:
_fla_build_cp_context = None

logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -342,14 +347,37 @@ def slice_log_prob_with_cp(
return torch.cat([chunk_1, chunk_2], dim=0)


def setup_hybrid_cp(model: nn.Module, cp_group: dist.ProcessGroup, cp_rank: int, cp_world_size: int) -> None:
"""Configure GatedDeltaNet modules for native fla CP instead of all-gather duplication.
def build_gdn_cp_context(module: nn.Module, cu_seqlens: torch.Tensor, device: torch.device):
"""Build fla CP context for a GatedDeltaNet module from packed sequence boundaries.

Args:
module: GDN module with ``cp_group`` / ``cp_world_size`` / ``conv_kernel_size``.
cu_seqlens: Global packed sequence boundaries (e.g. ``packed_seq_params.cu_seqlens_q``).
device: Target device.

Walks the model tree looking for HuggingfaceAttention submodules that have a
``linear_attn`` child (i.e. DeltaNet layers). For each one it sets the CP
metadata so that ``_build_cp_context`` produces a valid context, and flips
``hybrid_cp`` so the parent skips the all-gather path.
Returns ``None`` when CP is not configured on the module (``cp_group`` not set).
Raises ``RuntimeError`` if hybrid CP is configured but ``fla.ops.cp`` is missing.
"""
cp_group = getattr(module, "cp_group", None)
if cp_group is None:
return None
if _fla_build_cp_context is None:
raise RuntimeError(
"Hybrid CP requires fla.ops.cp (flash-linear-attention >= 0.4.2) " "but it could not be imported."
)
if cu_seqlens is None or cu_seqlens.numel() < 2:
raise ValueError(f"Hybrid CP requires valid cu_seqlens (at least 2 elements) but got {cu_seqlens}")
return _fla_build_cp_context(
cu_seqlens=cu_seqlens.to(device=device, dtype=torch.int32),
group=cp_group,
conv1d_kernel_size=module.conv_kernel_size,
)


def detect_and_setup_hybrid_cp(
model: nn.Module, cp_group: dist.ProcessGroup, cp_rank: int, cp_world_size: int
) -> None:
"""Scan for GatedDeltaNet modules and configure them for native fla CP."""
from miles_plugins.models.hf_attention import HuggingfaceAttention

count = 0
Expand All @@ -364,4 +392,4 @@ def setup_hybrid_cp(model: nn.Module, cp_group: dist.ProcessGroup, cp_rank: int,
count += 1

if count > 0:
logger.info(f"Configured hybrid CP on {count} DeltaNet modules (fla native state passing)")
logger.info(f"Configured hybrid CP on {count} GDN modules (fla native state passing)")
121 changes: 119 additions & 2 deletions miles_plugins/models/hf_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,105 @@ def _fix_dtype(d):
return ns


class _ZigzagSequentialExchange(torch.autograd.Function):
"""P2P exchange to convert between zigzag and sequential CP layouts.

For CP=2, zigzag rank 0 holds [sub_0, sub_3] and rank 1 holds [sub_1, sub_2].
Sequential rank 0 needs [sub_0, sub_1] and rank 1 needs [sub_2, sub_3].
This exchanges the misplaced sub-chunk between ranks via a single sendrecv.
"""

@staticmethod
def forward(ctx, send_buf, cp_group, cp_rank):
ctx.cp_group = cp_group
ctx.cp_rank = cp_rank
recv_buf = torch.empty_like(send_buf)
peer = 1 - cp_rank
if cp_rank == 0:
dist.send(send_buf.contiguous(), group_dst=peer, group=cp_group)
dist.recv(recv_buf, group_src=peer, group=cp_group)
else:
dist.recv(recv_buf, group_src=peer, group=cp_group)
dist.send(send_buf.contiguous(), group_dst=peer, group=cp_group)
return recv_buf

@staticmethod
def backward(ctx, grad_recv):
grad_send = torch.empty_like(grad_recv)
peer = 1 - ctx.cp_rank
if ctx.cp_rank == 0:
dist.send(grad_recv.contiguous(), group_dst=peer, group=ctx.cp_group)
dist.recv(grad_send, group_src=peer, group=ctx.cp_group)
else:
dist.recv(grad_send, group_src=peer, group=ctx.cp_group)
dist.send(grad_recv.contiguous(), group_dst=peer, group=ctx.cp_group)
return grad_send, None, None

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The _ZigzagSequentialExchange class and its associated logic are hardcoded for a Context Parallel (CP) world size of 2. Specifically, the peer calculation peer = 1 - cp_rank (line 54) and the assumption that each rank holds exactly two sub-chunks per sequence will fail or produce incorrect results if cp_world_size > 2. Since this logic is triggered whenever hybrid_cp is enabled and cp_world_size > 1, it will cause runtime errors or silent data corruption in configurations with CP > 2.

References
  1. Avoid hardcoding model dimensions or parameters; derive them from configuration or input tensor shapes instead.



def _zigzag_to_sequential(hidden_states, local_cu_seqlens, cp_group, cp_rank):
"""Convert zigzag CP layout to sequential via P2P exchange (CP=2 only).

Rank 0 zigzag: [sub_0, sub_3] → sequential: [sub_0, sub_1]
Rank 1 zigzag: [sub_1, sub_2] → sequential: [sub_2, sub_3]
"""
# Split each sample into ascending (first half) and descending (second half)
keep_parts, send_parts = [], []
for i in range(len(local_cu_seqlens) - 1):
start, end = local_cu_seqlens[i], local_cu_seqlens[i + 1]
mid = (start + end) // 2
if cp_rank == 0:
keep_parts.append(hidden_states[start:mid]) # sub_0
send_parts.append(hidden_states[mid:end]) # sub_3 → send to rank 1
else:
send_parts.append(hidden_states[start:mid]) # sub_1 → send to rank 0
keep_parts.append(hidden_states[mid:end]) # sub_2

send_buf = torch.cat(send_parts, dim=0)
recv_buf = _ZigzagSequentialExchange.apply(send_buf, cp_group, cp_rank)

# Reassemble: both ranks → [keep, recv]
result = []
offset = 0
for i in range(len(local_cu_seqlens) - 1):
chunk_len = (local_cu_seqlens[i + 1] - local_cu_seqlens[i]) // 2
result.append(keep_parts[i])
result.append(recv_buf[offset : offset + chunk_len])
offset += chunk_len
return torch.cat(result, dim=0)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The _zigzag_to_sequential function (and similarly _sequential_to_zigzag) does not correctly handle sequences with odd lengths. The use of integer division mid = (start + end) // 2 (line 86) and chunk_len = ... // 2 (line 101) assumes that each sample is split into two equal halves. If a sample has an odd number of tokens in the local buffer (e.g., 5 tokens), mid will be 2, resulting in 2 tokens kept and 3 tokens sent. However, chunk_len will be 2, causing the reassembly logic to slice only 2 tokens from the received buffer, losing the 3rd token and corrupting the offsets for all subsequent samples in the packed batch.



def _sequential_to_zigzag(hidden_states, local_cu_seqlens, cp_group, cp_rank):
"""Convert sequential CP layout back to zigzag via P2P exchange (CP=2 only).

Rank 0 sequential: [sub_0, sub_1] → zigzag: [sub_0, sub_3]
Rank 1 sequential: [sub_2, sub_3] → zigzag: [sub_1, sub_2]
"""
keep_parts, send_parts = [], []
for i in range(len(local_cu_seqlens) - 1):
start, end = local_cu_seqlens[i], local_cu_seqlens[i + 1]
mid = (start + end) // 2
keep_parts.append(hidden_states[start:mid]) # first half
send_parts.append(hidden_states[mid:end]) # second half

send_buf = torch.cat(send_parts, dim=0)
recv_buf = _ZigzagSequentialExchange.apply(send_buf, cp_group, cp_rank)

# Rank 0: [keep (sub_0), recv (sub_3)]
# Rank 1: [recv (sub_1), keep (sub_2)]
result = []
offset = 0
for i in range(len(local_cu_seqlens) - 1):
chunk_len = (local_cu_seqlens[i + 1] - local_cu_seqlens[i]) // 2
if cp_rank == 0:
result.append(keep_parts[i])
result.append(recv_buf[offset : offset + chunk_len])
else:
result.append(recv_buf[offset : offset + chunk_len])
result.append(keep_parts[i])
offset += chunk_len
return torch.cat(result, dim=0)


class _AllGatherForDuplicatedComputation(torch.autograd.Function):
"""All-gather whose backward just returns the local gradient slice (no reduce).

Expand Down Expand Up @@ -119,7 +218,17 @@ def forward(
group=mpu.get_tensor_model_parallel_group(),
)

if mpu.get_context_parallel_world_size() > 1 and not self.hybrid_cp:
if mpu.get_context_parallel_world_size() > 1 and self.hybrid_cp:
cp_size = mpu.get_context_parallel_world_size()
local_cu_seqlens = cu_seqlens // cp_size

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Calculating local_cu_seqlens = cu_seqlens // cp_size assumes that every sequence boundary in the packed batch is a multiple of cp_size. In a packed sequence scenario, individual sequences can have arbitrary lengths. If a sequence length is not a multiple of cp_size, the calculated local boundaries will not align with the actual data distribution across CP ranks, leading to incorrect slicing in _zigzag_to_sequential and potential loss of tokens.

hidden_states = _zigzag_to_sequential(
hidden_states,
local_cu_seqlens,
mpu.get_context_parallel_group(),
mpu.get_context_parallel_rank(),
)

elif mpu.get_context_parallel_world_size() > 1:
cp_size = mpu.get_context_parallel_world_size()
# Use custom all-gather whose backward returns local gradient
# instead of reduce-scatter, since the computation is duplicated.
Expand Down Expand Up @@ -154,7 +263,15 @@ def forward(

output = output.permute(1, 0, 2) # [seq_len, bsz, hidden_dim]

if mpu.get_context_parallel_world_size() > 1 and not self.hybrid_cp:
if mpu.get_context_parallel_world_size() > 1 and self.hybrid_cp:
output = _sequential_to_zigzag(
output,
local_cu_seqlens,
mpu.get_context_parallel_group(),
mpu.get_context_parallel_rank(),
)

elif mpu.get_context_parallel_world_size() > 1:
cp_rank = mpu.get_context_parallel_rank()
output_list = []
for i in range(len(cu_seqlens) - 1):
Expand Down
21 changes: 2 additions & 19 deletions miles_plugins/models/qwen3_5.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,7 @@
except ImportError:
pass

try:
from fla.ops.cp import FLACPContext, build_cp_context
except ImportError:
FLACPContext = None
build_cp_context = None
from miles.backends.training_utils.cp_utils import build_gdn_cp_context

from .hf_attention import HuggingfaceAttention, _load_hf_config

Expand Down Expand Up @@ -87,27 +83,14 @@ def __init__(self, config, layer_idx: int):

self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)

def _build_cp_context(self, local_seq_len: int, device: torch.device):
"""Build fla CP context from the local (sharded) sequence length."""
cp_group = getattr(self, "cp_group", None)
if cp_group is None or build_cp_context is None:
return None
global_seq_len = local_seq_len * self.cp_world_size
global_cu_seqlens = torch.tensor([0, global_seq_len], dtype=torch.int32, device=device)
return build_cp_context(
cu_seqlens=global_cu_seqlens,
group=cp_group,
conv1d_kernel_size=self.conv_kernel_size,
)

def forward(
self,
hidden_states: torch.Tensor,
cu_seqlens: torch.Tensor = None,
):
batch_size, seq_len, _ = hidden_states.shape

cp_context = self._build_cp_context(seq_len, hidden_states.device)
cp_context = build_gdn_cp_context(self, cu_seqlens, hidden_states.device)

# Projections (flat layout: [Q_all, K_all, V_all])
mixed_qkv = self.in_proj_qkv(hidden_states)
Expand Down
21 changes: 2 additions & 19 deletions miles_plugins/models/qwen3_next.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,7 @@
except ImportError:
pass

try:
from fla.ops.cp import FLACPContext, build_cp_context
except ImportError:
FLACPContext = None
build_cp_context = None
from miles.backends.training_utils.cp_utils import build_gdn_cp_context

from .hf_attention import HuggingfaceAttention

Expand Down Expand Up @@ -80,19 +76,6 @@ def __init__(self, config, layer_idx: int):

self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)

def _build_cp_context(self, local_seq_len: int, device: torch.device):
"""Build fla CP context from the local (sharded) sequence length."""
cp_group = getattr(self, "cp_group", None)
if cp_group is None or build_cp_context is None:
return None
global_seq_len = local_seq_len * self.cp_world_size
global_cu_seqlens = torch.tensor([0, global_seq_len], dtype=torch.int32, device=device)
return build_cp_context(
cu_seqlens=global_cu_seqlens,
group=cp_group,
conv1d_kernel_size=self.conv_kernel_size,
)

def fix_query_key_value_ordering(self, mixed_qkvz, mixed_ba):
"""
Derives `query`, `key` and `value` tensors from `mixed_qkvz` and `mixed_ba`.
Expand Down Expand Up @@ -127,7 +110,7 @@ def forward(
hidden_states: torch.Tensor,
cu_seqlens: torch.Tensor = None,
):
cp_context = self._build_cp_context(hidden_states.shape[1], hidden_states.device)
cp_context = build_gdn_cp_context(self, cu_seqlens, hidden_states.device)

projected_states_qkvz = self.in_proj_qkvz(hidden_states)
projected_states_ba = self.in_proj_ba(hidden_states)
Expand Down
1 change: 1 addition & 0 deletions tests/fast/backends/training_utils/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

Loading