Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 53 additions & 23 deletions atom/model_ops/attention_mha.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
)
from .attention_mla import MLAModules
from aiter.ops.triton.unified_attention import unified_attention
from aiter.ops.triton.fused_kv_cache import fused_qk_rope_reshape_and_cache


class Attention(nn.Module):
Expand All @@ -30,6 +31,7 @@ def __init__(
mla_modules: Optional[MLAModules] = None,
sinks: Optional[nn.Parameter] = None,
sliding_window: Optional[int] = None,
rotary_emb: Optional[torch.nn.Module] = None,
**kwargs,
):
super().__init__()
Expand All @@ -47,6 +49,7 @@ def __init__(
self.sliding_window = (
(sliding_window - 1, 0) if sliding_window is not None else (-1, -1)
)
self.rotary_emb = rotary_emb

def forward(
self,
Expand Down Expand Up @@ -82,27 +85,58 @@ def forward(
k_cache = v_cache = torch.tensor([])
k_scale = v_scale = None

assert self.rotary_emb is None or (self.rotary_emb is not None and position is not None)
if k_cache.numel() and v_cache.numel():
if use_triton_unified_attention:
aiter.reshape_and_cache_flash(
k,
v,
k_cache.view(
k_cache.shape[0], -1, self.num_kv_heads, self.head_dim
),
v_cache.view(
v_cache.shape[0], -1, self.num_kv_heads, self.head_dim
),
attn_metadata.slot_mapping,
(
self.kv_cache_dtype
if self.kv_cache_dtype.startswith("fp8")
else "auto"
),
self.one_scale,
self.one_scale,
k_scale = v_scale = self.one_scale
k_cache = k_cache.view(
k_cache.shape[0], -1, self.num_kv_heads, self.head_dim
)
v_cache = v_cache.view(
v_cache.shape[0], -1, self.num_kv_heads, self.head_dim
)
if context.is_prefill or self.rotary_emb is None:
if self.rotary_emb is not None:
q, k = self.rotary_emb(position, q, k)
aiter.reshape_and_cache_flash(
k,
v,
k_cache,
v_cache,
attn_metadata.slot_mapping,
(
self.kv_cache_dtype
if self.kv_cache_dtype.startswith("fp8")
else "auto"
),
k_scale,
v_scale,
)
else:
q, k, k_cache, v_cache = fused_qk_rope_reshape_and_cache(
q,
k,
v,
k_cache,
v_cache,
attn_metadata.slot_mapping,
position,
self.rotary_emb.cos_cache,
self.rotary_emb.sin_cache,
k_scale,
v_scale,
self.rotary_emb.is_neox_style,
flash_layout=True,
apply_scale=self.kv_cache_dtype.startswith("fp8"),
offs=None,
q_out=q,
k_out=k,
output_zeros=False,
)
else:
if self.rotary_emb is not None:
assert position is not None
q, k = self.rotary_emb(position, q, k)
if self.kv_cache_dtype == "fp8":
aiter.reshape_and_cache_with_pertoken_quant(
k,
Expand Down Expand Up @@ -133,12 +167,8 @@ def forward(
if k_cache.numel() and v_cache.numel():
unified_attention(
q,
k_cache.view(
k_cache.shape[0], -1, self.num_kv_heads, self.head_dim
),
v_cache.view(
v_cache.shape[0], -1, self.num_kv_heads, self.head_dim
),
k_cache,
v_cache,
o,
cu_seqlens_q=attn_metadata.cu_seqlens_q,
seqused_k=attn_metadata.context_lens,
Expand Down
2 changes: 2 additions & 0 deletions atom/model_ops/base_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ def __init__(
mla_modules: Optional[MLAModules] = None,
sinks: Optional[nn.Parameter] = None,
per_layer_sliding_window: Optional[int] = None,
rotary_emb: Optional[torch.nn.Module] = None,
**kwargs,
):
super().__init__()
Expand Down Expand Up @@ -98,6 +99,7 @@ def __init__(
mla_modules,
sinks=sinks,
sliding_window=per_layer_sliding_window,
rotary_emb=rotary_emb,
)

compilation_config = atom_config.compilation_config
Expand Down
11 changes: 7 additions & 4 deletions atom/models/gpt_oss.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,15 +123,16 @@ def __init__(
prefix=f"{prefix}.attn",
sinks=self.sinks,
layer_num=self.layer_idx,
rotary_emb=self.rotary_emb,
)

def forward(
self, hidden_states: torch.Tensor, positions: torch.Tensor
) -> torch.Tensor:
qkv = self.qkv_proj(hidden_states)
q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
# q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v, positions)
output = self.o_proj(attn_output)
return output

Expand Down Expand Up @@ -179,7 +180,7 @@ def __init__(
def forward(self, x: torch.Tensor) -> torch.Tensor:
num_tokens = x.shape[0]

g = self.router(x[..., :self.hidden_size])
g = self.router(x[..., : self.hidden_size])
x = self.experts(hidden_states=x, router_logits=g)

if self.is_sequence_parallel:
Expand Down Expand Up @@ -212,7 +213,9 @@ def __init__(
)
self.mlp = MLPBlock(atom_config, self.layer_idx, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNorm(config.hidden_size, eps=1e-5)
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=1e-5, x_pad_to_multiple=256)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=1e-5, x_pad_to_multiple=256
)

def forward(
self,
Expand Down