Integrate fused flashinfer rope_quantize_fp8_append_paged_kv_cache kernel

python/sglang/srt/layers/attention/base_attn_backend.py

@@ -53,6 +53,12 @@ def init_forward_metadata_replay_cuda_graph(
         """Init the metadata for a forward pass for replaying a cuda graph."""
         raise NotImplementedError()
 
+    def update_fused_rope_metadata_for_replay(
+        self, bs: int, out_cache_loc: Optional[torch.Tensor]
+    ):
+        """Update fused RoPE metadata for CUDA graph replay. No-op by default."""
+        pass
+
     def get_cuda_graph_seq_len_fill_value(self):
         """Get the fill value for padded seq lens. Typically, it is 0 or 1."""
         raise NotImplementedError()

python/sglang/srt/layers/attention/hybrid_attn_backend.py

@@ -108,6 +108,9 @@ def init_forward_metadata_replay_cuda_graph(
             seq_lens_cpu,
         )
 
+    def update_fused_rope_metadata_for_replay(self, bs, out_cache_loc):
+        self.decode_backend.update_fused_rope_metadata_for_replay(bs, out_cache_loc)
+
     def get_cuda_graph_seq_len_fill_value(self):
         return self.decode_backend.get_cuda_graph_seq_len_fill_value()
 

python/sglang/srt/layers/attention/tbo_backend.py

@@ -173,6 +173,9 @@ def _init_forward_metadata_cuda_graph_children(
         getattr(child_left, fn_name)(**args_left)
         getattr(child_right, fn_name)(**args_right)
 
+    def update_fused_rope_metadata_for_replay(self, bs, out_cache_loc):
+        self.primary.update_fused_rope_metadata_for_replay(bs, out_cache_loc)
+
     def get_cuda_graph_seq_len_fill_value(self):
         ans = self.primary.get_cuda_graph_seq_len_fill_value()
         for child in self.children: