From 439942c4382a88341f458b9a054d513870ceb08e Mon Sep 17 00:00:00 2001 From: chenjun Date: Wed, 26 Nov 2025 08:54:03 +0000 Subject: [PATCH] add fused_qk_rope_reshape_and_cache --- atom/model_ops/attention_mha.py | 76 ++++++++++++++++++++++---------- atom/model_ops/base_attention.py | 2 + atom/models/gpt_oss.py | 11 +++-- 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/atom/model_ops/attention_mha.py b/atom/model_ops/attention_mha.py index fe6b2d0026..ceb851c082 100644 --- a/atom/model_ops/attention_mha.py +++ b/atom/model_ops/attention_mha.py @@ -15,6 +15,7 @@ ) from .attention_mla import MLAModules from aiter.ops.triton.unified_attention import unified_attention +from aiter.ops.triton.fused_kv_cache import fused_qk_rope_reshape_and_cache class Attention(nn.Module): @@ -30,6 +31,7 @@ def __init__( mla_modules: Optional[MLAModules] = None, sinks: Optional[nn.Parameter] = None, sliding_window: Optional[int] = None, + rotary_emb: Optional[torch.nn.Module] = None, **kwargs, ): super().__init__() @@ -47,6 +49,7 @@ def __init__( self.sliding_window = ( (sliding_window - 1, 0) if sliding_window is not None else (-1, -1) ) + self.rotary_emb = rotary_emb def forward( self, @@ -82,27 +85,58 @@ def forward( k_cache = v_cache = torch.tensor([]) k_scale = v_scale = None + assert self.rotary_emb is None or (self.rotary_emb is not None and position is not None) if k_cache.numel() and v_cache.numel(): if use_triton_unified_attention: - aiter.reshape_and_cache_flash( - k, - v, - k_cache.view( - k_cache.shape[0], -1, self.num_kv_heads, self.head_dim - ), - v_cache.view( - v_cache.shape[0], -1, self.num_kv_heads, self.head_dim - ), - attn_metadata.slot_mapping, - ( - self.kv_cache_dtype - if self.kv_cache_dtype.startswith("fp8") - else "auto" - ), - self.one_scale, - self.one_scale, + k_scale = v_scale = self.one_scale + k_cache = k_cache.view( + k_cache.shape[0], -1, self.num_kv_heads, self.head_dim ) + v_cache = v_cache.view( + v_cache.shape[0], -1, self.num_kv_heads, self.head_dim + ) + if context.is_prefill or self.rotary_emb is None: + if self.rotary_emb is not None: + q, k = self.rotary_emb(position, q, k) + aiter.reshape_and_cache_flash( + k, + v, + k_cache, + v_cache, + attn_metadata.slot_mapping, + ( + self.kv_cache_dtype + if self.kv_cache_dtype.startswith("fp8") + else "auto" + ), + k_scale, + v_scale, + ) + else: + q, k, k_cache, v_cache = fused_qk_rope_reshape_and_cache( + q, + k, + v, + k_cache, + v_cache, + attn_metadata.slot_mapping, + position, + self.rotary_emb.cos_cache, + self.rotary_emb.sin_cache, + k_scale, + v_scale, + self.rotary_emb.is_neox_style, + flash_layout=True, + apply_scale=self.kv_cache_dtype.startswith("fp8"), + offs=None, + q_out=q, + k_out=k, + output_zeros=False, + ) else: + if self.rotary_emb is not None: + assert position is not None + q, k = self.rotary_emb(position, q, k) if self.kv_cache_dtype == "fp8": aiter.reshape_and_cache_with_pertoken_quant( k, @@ -133,12 +167,8 @@ def forward( if k_cache.numel() and v_cache.numel(): unified_attention( q, - k_cache.view( - k_cache.shape[0], -1, self.num_kv_heads, self.head_dim - ), - v_cache.view( - v_cache.shape[0], -1, self.num_kv_heads, self.head_dim - ), + k_cache, + v_cache, o, cu_seqlens_q=attn_metadata.cu_seqlens_q, seqused_k=attn_metadata.context_lens, diff --git a/atom/model_ops/base_attention.py b/atom/model_ops/base_attention.py index 3f95f2fff0..385b3cceda 100644 --- a/atom/model_ops/base_attention.py +++ b/atom/model_ops/base_attention.py @@ -62,6 +62,7 @@ def __init__( mla_modules: Optional[MLAModules] = None, sinks: Optional[nn.Parameter] = None, per_layer_sliding_window: Optional[int] = None, + rotary_emb: Optional[torch.nn.Module] = None, **kwargs, ): super().__init__() @@ -98,6 +99,7 @@ def __init__( mla_modules, sinks=sinks, sliding_window=per_layer_sliding_window, + rotary_emb=rotary_emb, ) compilation_config = atom_config.compilation_config diff --git a/atom/models/gpt_oss.py b/atom/models/gpt_oss.py index b562fd94e3..527c508f1c 100644 --- a/atom/models/gpt_oss.py +++ b/atom/models/gpt_oss.py @@ -123,6 +123,7 @@ def __init__( prefix=f"{prefix}.attn", sinks=self.sinks, layer_num=self.layer_idx, + rotary_emb=self.rotary_emb, ) def forward( @@ -130,8 +131,8 @@ def forward( ) -> torch.Tensor: qkv = self.qkv_proj(hidden_states) q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1) - q, k = self.rotary_emb(positions, q, k) - attn_output = self.attn(q, k, v) + # q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, positions) output = self.o_proj(attn_output) return output @@ -179,7 +180,7 @@ def __init__( def forward(self, x: torch.Tensor) -> torch.Tensor: num_tokens = x.shape[0] - g = self.router(x[..., :self.hidden_size]) + g = self.router(x[..., : self.hidden_size]) x = self.experts(hidden_states=x, router_logits=g) if self.is_sequence_parallel: @@ -212,7 +213,9 @@ def __init__( ) self.mlp = MLPBlock(atom_config, self.layer_idx, prefix=f"{prefix}.mlp") self.input_layernorm = RMSNorm(config.hidden_size, eps=1e-5) - self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=1e-5, x_pad_to_multiple=256) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=1e-5, x_pad_to_multiple=256 + ) def forward( self,