Add model gpt-oss

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "3rdparty/triton"]
+	path = 3rdparty/triton
+	url = https://github.com/dongfengy/triton.git
+	branch = fused_moe_triton_0613

python/sglang/bench_offline_throughput.py

@@ -234,6 +234,8 @@ def throughput_test_once(
 
     st = time.perf_counter()
     gen_out = backend.generate(prompt=prompt, sampling_params=sampling_params)
+    print(f"prompt: {prompt}")
+    print(f"gen_out: {gen_out}")
     latency = time.perf_counter() - st
 
     if profile:

python/sglang/srt/layers/activation.py

@@ -138,6 +138,21 @@ def forward_hip(self, x: torch.Tensor) -> torch.Tensor:
         return out
 
 
+class SwiGLU(CustomOp):
+    def forward_native(self, x: torch.Tensor, alpha: float = 1.702, pair_wise: bool = True) -> torch.Tensor:
+        # reference implementation
+        if not pair_wise:
+            x_glu, x_linear = torch.chunk(x, 2, dim=-1)
+        else:
+            x_glu, x_linear = x[..., ::2], x[..., 1::2]
+        out_glu = x_glu * torch.sigmoid(alpha * x_glu)
+        return out_glu * (x_linear + 1) # Note that here add an extra bias of 1 to the linear layer
+
+    def forward_cuda(self, x: torch.Tensor, alpha: float = 1.702, pair_wise: bool = True) -> torch.Tensor:
+        # TODO: Implement the CUDA kernel for SwiGLU in sgl-kernel
+        return self.forward_native(x, alpha, pair_wise)
+
+
 class ScaledActivation(nn.Module):
     """An activation function with post-scale parameters.
 

python/sglang/srt/layers/attention/torch_native_backend.py

@@ -268,3 +268,324 @@ def forward_decode(
 
     def support_triton(self):
         return False
+
+
+class TorchNativeAttnSinkBackend(TorchNativeAttnBackend):
+    def __init__(self, model_runner: ModelRunner):
+        super().__init__(model_runner)
+        self.forward_metadata = None
+        self.device = model_runner.device
+
+    @staticmethod
+    def _scaled_dot_product_attention(Q, K, V, S, scaling, sliding_window):
+        # sliding_window <= 0 means no sliding window
+        # Q: [n_tokens_q, n_heads, q_mult, d_head]
+        # K: [n_tokens_kv, n_heads, d_head]
+        # V: [n_tokens_kv, n_heads, d_head]
+        n_tokens_q, n_heads, q_mult, d_head = Q.shape
+        n_tokens_kv = K.shape[0]
+
+        assert K.shape == (n_tokens_kv, n_heads, d_head)
+        assert V.shape == (n_tokens_kv, n_heads, d_head)
+
+        K = K[:, :, None, :].expand(-1, -1, q_mult, -1)
+        V = V[:, :, None, :].expand(-1, -1, q_mult, -1)
+        S = S.reshape(n_heads, q_mult, 1, 1).expand(-1, -1, n_tokens_q, -1)
+
+        if n_tokens_q == n_tokens_kv:  # Prefill
+            mask = torch.triu(
+                Q.new_full((n_tokens_q, n_tokens_kv), -float("inf")), diagonal=1
+            )
+        else:  # Decode
+            mask = Q.new_zeros((n_tokens_q, n_tokens_kv))
+
+        if sliding_window is not None and sliding_window > 0:
+            mask += torch.tril(
+                mask.new_full((n_tokens_q, n_tokens_kv), -float("inf")),
+                diagonal=n_tokens_kv - n_tokens_q - sliding_window,
+            )
+
+        QK = torch.einsum("qhmd,khmd->hmqk", Q, K)
+        QK *= scaling
+        QK += mask[None, None, :, :]
+        QK = torch.cat([QK, S], dim=-1)
+
+        W = torch.softmax(QK, dim=-1)
+        W = W[..., :-1]
+
+        attn = torch.einsum("hmqk,khmd->qhmd", W, V)
+
+        return attn.reshape(n_tokens_q, -1)
+
+    def _run_sdpa_forward_extend(
+        self,
+        query: torch.Tensor,
+        output: torch.Tensor,
+        k_cache: torch.Tensor,
+        v_cache: torch.Tensor,
+        req_to_token: torch.Tensor,
+        req_pool_indices: torch.Tensor,
+        seq_lens: torch.Tensor,
+        extend_prefix_lens: torch.Tensor,
+        extend_seq_lens: torch.Tensor,
+        num_kv_heads: int,
+        q_mult: int,
+        scaling=None,
+        sliding_window=None,
+        attention_sinks=None,
+        enable_gqa=False,
+        causal=False,
+    ):
+        """Run the extend forward by using custom sdpa op.
+
+        Args:
+            query: [num_tokens, num_q_heads, head_size]
+            output: [num_tokens, num_q_heads, head_size]
+            k_cache: [max_total_num_tokens, num_kv_heads, head_size]
+            v_cache: [max_total_num_tokens, num_kv_heads, head_size]
+            req_to_token: [max_num_reqs, max_context_len]
+            req_pool_indices: [num_seqs]
+            seq_lens: [num_seqs]
+            extend_prefix_lens: [num_seqs]
+            extend_seq_lens: [num_seqs]
+            num_kv_heads: int
+            q_mult: int
+            scaling: float or None
+            sliding_window: int or None
+            attention_sinks: torch.Tensor or None
+            enable_gqa: bool
+            causal: bool
+
+        Returns:
+            output: [num_tokens, num_q_heads, head_size]
+        """
+
+        assert seq_lens.shape[0] == extend_prefix_lens.shape[0]
+        assert seq_lens.shape[0] == extend_seq_lens.shape[0]
+
+        # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size]
+        query = query.movedim(0, query.dim() - 2)
+
+        start_q, start_kv = 0, 0
+        for seq_idx in range(seq_lens.shape[0]):
+            # TODO: this loop process a sequence per iter, this is inefficient.
+            # Need optimize the performance later.
+
+            extend_seq_len_q = extend_seq_lens[seq_idx]
+            prefill_seq_len_q = extend_prefix_lens[seq_idx]
+
+            seq_len_kv = seq_lens[seq_idx]
+            end_q = start_q + extend_seq_len_q
+            end_kv = start_kv + seq_len_kv
+
+            per_req_query = query[:, start_q:end_q, :]
+            per_req_query_redudant = torch.empty(
+                (per_req_query.shape[0], seq_len_kv, per_req_query.shape[2]),
+                dtype=per_req_query.dtype,
+                device=per_req_query.device,
+            )
+
+            per_req_query_redudant[:, prefill_seq_len_q:, :] = per_req_query
+
+            # get key and value from cache. per_req_tokens contains the kv cache
+            # index for each token in the sequence.
+            req_pool_idx = req_pool_indices[seq_idx]
+            per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv]
+            per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2)
+            per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2)
+
+            per_req_query_redudant = per_req_query_redudant.permute(1, 0, 2).reshape(
+                seq_len_kv, num_kv_heads, q_mult, per_req_query_redudant.shape[-1]
+            )
+            per_req_key = per_req_key.permute(1, 0, 2)
+            per_req_value = per_req_value.permute(1, 0, 2)
+
+            per_req_out_redudant = TorchNativeAttnSinkBackend._scaled_dot_product_attention(
+                per_req_query_redudant,
+                per_req_key,
+                per_req_value,
+                attention_sinks,
+                scaling=scaling,
+                sliding_window=sliding_window,
+            ).reshape(seq_len_kv, -1, per_req_value.shape[-1])
+            output[start_q:end_q, :, :] = per_req_out_redudant[prefill_seq_len_q:, :, :]
+            start_q, start_kv = end_q, end_kv
+        return output
+
+    def _run_sdpa_forward_decode(
+        self,
+        query: torch.Tensor,
+        output: torch.Tensor,
+        k_cache: torch.Tensor,
+        v_cache: torch.Tensor,
+        req_to_token: torch.Tensor,
+        req_pool_indices: torch.Tensor,
+        seq_lens: torch.Tensor,
+        num_kv_heads: int,
+        q_mult: int,
+        scaling=None,
+        sliding_window=None,
+        attention_sinks=None,
+        enable_gqa=False,
+        causal=False,
+    ):
+        """Run the decode forward by using custom sdpa op.
+
+        Args:
+            query: [num_tokens, num_q_heads, head_size]
+            output: [num_tokens, num_q_heads, head_size]
+            k_cache: [max_total_num_tokens, num_kv_heads, head_size]
+            v_cache: [max_total_num_tokens, num_kv_heads, head_size]
+            req_to_token: [max_num_reqs, max_context_len]
+            req_pool_indices: [num_seqs]
+            seq_lens: [num_seqs]
+            num_kv_heads: int
+            q_mult: int
+            scaling: float or None
+            sliding_window: int or None
+            attention_sinks: torch.Tensor or None
+            enable_gqa: bool
+            causal: bool
+
+        Returns:
+            output: [num_tokens, num_q_heads, head_size]
+        """
+
+        # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size]
+        query = query.movedim(0, query.dim() - 2)
+
+        start_q, start_kv = 0, 0
+        for seq_idx in range(seq_lens.shape[0]):
+            # TODO: this loop process a sequence per iter, this is inefficient.
+            # Need optimize the performance later.
+
+            seq_len_q = 1
+            seq_len_kv = seq_lens[seq_idx]
+            end_q = start_q + seq_len_q
+            end_kv = start_kv + seq_len_kv
+
+            per_req_query = query[:, start_q:end_q, :]
+
+            # get key and value from cache. per_req_tokens contains the kv cache
+            # index for each token in the sequence.
+            req_pool_idx = req_pool_indices[seq_idx]
+            per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv]
+
+            per_req_query = per_req_query.permute(1, 0, 2).reshape(
+                seq_len_q, num_kv_heads, q_mult, per_req_query.shape[-1]
+            )
+            per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2)
+            per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2)
+            per_req_key = per_req_key.permute(1, 0, 2)
+            per_req_value = per_req_value.permute(1, 0, 2)
+
+            per_req_out = (
+                TorchNativeAttnSinkBackend._scaled_dot_product_attention(
+                    per_req_query,
+                    per_req_key,
+                    per_req_value,
+                    attention_sinks,
+                    scaling=scaling,
+                    sliding_window=sliding_window,
+                )
+                .reshape(seq_len_q, -1, per_req_value.shape[-1])
+            )
+            output[start_q:end_q, :, :] = per_req_out
+            start_q, start_kv = end_q, end_kv
+
+        return output
+
+    def forward_extend(
+        self,
+        q,
+        k,
+        v,
+        layer: RadixAttention,
+        forward_batch: ForwardBatch,
+        save_kv_cache=True,
+    ):
+        if layer.qk_head_dim != layer.v_head_dim:
+            o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim))
+        else:
+            o = torch.empty_like(q)
+
+        if save_kv_cache:
+            forward_batch.token_to_kv_pool.set_kv_buffer(
+                layer, forward_batch.out_cache_loc, k, v
+            )
+
+        use_gqa = layer.tp_q_head_num != layer.tp_k_head_num
+
+        q_ = q.view(-1, layer.tp_q_head_num, layer.qk_head_dim)
+        o_ = o.view(-1, layer.tp_q_head_num, layer.v_head_dim)
+
+        causal = True
+        if layer.is_cross_attention or layer.attn_type == AttentionType.ENCODER_ONLY:
+            causal = False
+
+        self._run_sdpa_forward_extend(
+            q_,
+            o_,
+            forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
+            forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
+            forward_batch.req_to_token_pool.req_to_token,
+            forward_batch.req_pool_indices,
+            forward_batch.seq_lens,
+            forward_batch.extend_prefix_lens,
+            forward_batch.extend_seq_lens,
+            layer.tp_k_head_num,
+            layer.tp_q_head_num // layer.tp_k_head_num,
+            scaling=layer.scaling,
+            sliding_window=layer.sliding_window_size + 1, # torch native attn sink uses sliding window without -1
+            attention_sinks=layer.attention_sinks,
+            enable_gqa=use_gqa,
+            causal=causal,
+        )
+        return o
+
+    def forward_decode(
+        self,
+        q,
+        k,
+        v,
+        layer: RadixAttention,
+        forward_batch: ForwardBatch,
+        save_kv_cache=True,
+    ):
+        # During torch.compile, there is a bug in rotary_emb that causes the
+        # output value to have a 3D tensor shape. This reshapes the output correctly.
+        q = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim)
+
+        if layer.qk_head_dim != layer.v_head_dim:
+            o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim))
+        else:
+            o = torch.empty_like(q)
+
+        if save_kv_cache:
+            forward_batch.token_to_kv_pool.set_kv_buffer(
+                layer, forward_batch.out_cache_loc, k, v
+            )
+
+        use_gqa = layer.tp_q_head_num != layer.tp_k_head_num
+
+        q_ = q.view(-1, layer.tp_q_head_num, layer.qk_head_dim)
+        o_ = o.view(-1, layer.tp_q_head_num, layer.v_head_dim)
+
+        self._run_sdpa_forward_decode(
+            q_,
+            o_,
+            forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id),
+            forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id),
+            forward_batch.req_to_token_pool.req_to_token,
+            forward_batch.req_pool_indices,
+            forward_batch.seq_lens,
+            layer.tp_k_head_num,
+            layer.tp_q_head_num // layer.tp_k_head_num,
+            scaling=layer.scaling,
+            sliding_window=layer.sliding_window_size + 1, # torch native attn sink uses sliding window without -1
+            attention_sinks=layer.attention_sinks,
+            enable_gqa=use_gqa,
+            causal=False,
+        )
+
+        return o

python/sglang/srt/layers/moe/fused_moe_triton/__init__.py

@@ -14,7 +14,6 @@
 
 _config: Optional[Dict[str, Any]] = None
 
-
 @contextmanager
 def override_config(config):
     global _config
@@ -30,6 +29,7 @@ def get_config() -> Optional[Dict[str, Any]]:
 
 __all__ = [
     "FusedMoE",
+    "FusedMoEMethodBase",
     "FusedMoeWeightScaleSupported",
     "override_config",
     "get_config",