Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion csrc/libtorch_stable/moe/moe_ops.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ void topk_softplus_sqrt(
const std::optional<torch::stable::Tensor>& correction_bias,
const std::optional<torch::stable::Tensor>& input_ids,
const std::optional<torch::stable::Tensor>& tid2eid,
const std::optional<torch::stable::Tensor>& is_padding);
const std::optional<torch::stable::Tensor>& is_padding,
const std::optional<torch::stable::Tensor>& bias_vl,
int64_t image_sentinel_lo);

void moe_sum(torch::stable::Tensor& input, torch::stable::Tensor& output,
std::optional<torch::stable::Tensor> topk_ids,
Expand Down
200 changes: 172 additions & 28 deletions csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion csrc/libtorch_stable/moe/torch_bindings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
"topk_softplus_sqrt(Tensor! topk_weights, Tensor! topk_indices, Tensor! "
"token_expert_indices, Tensor gating_output, bool renormalize, float "
"routed_scaling_factor, Tensor? "
"bias, Tensor? input_ids, Tensor? tid2eid, Tensor? is_padding) -> ()");
"bias, Tensor? input_ids, Tensor? tid2eid, Tensor? is_padding, "
"Tensor? bias_vl=None, int image_sentinel_lo=0) -> ()");

// Calculate the result of moe by summing up the partial results
// from all selected experts. topk_ids/expert_map are optional and, when
Expand Down
1 change: 1 addition & 0 deletions docs/models/supported_models.md
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | | ✅︎ |
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
Expand Down
31 changes: 31 additions & 0 deletions tests/config/test_model_arch_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,3 +465,34 @@ def test_draft_model_arch_config(
_assert_model_config_methods(
model_config, expected, check_head_size=check_head_size
)


def test_deepseek_v4_convertor_splits_vision_architecture():
"""The vision checkpoint shares model_type/architectures with the
text-only DeepSeek-V4; the convertor routes it to the VL wrapper class
by rewriting hf_config.architectures (model-class resolution reads the
raw attribute)."""
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
from vllm.transformers_utils.model_arch_config_convertor import (
DeepseekV4ModelArchConfigConvertor,
)

text_cfg = DeepseekV4Config(architectures=["DeepseekV4ForCausalLM"])
conv = DeepseekV4ModelArchConfigConvertor(text_cfg, text_cfg)
assert conv.get_architectures() == ["DeepseekV4ForCausalLM"]
assert text_cfg.architectures == ["DeepseekV4ForCausalLM"]

vl_cfg = DeepseekV4Config(
architectures=["DeepseekV4ForCausalLM"], vision_n_layers=32
)
conv = DeepseekV4ModelArchConfigConvertor(vl_cfg, vl_cfg)
assert conv.get_architectures() == ["DeepseekV4ForConditionalGeneration"]
assert vl_cfg.architectures == ["DeepseekV4ForConditionalGeneration"]

# Speculative-draft configs (set by SpeculativeConfig.hf_config_override)
# keep their own architecture even with a vision tower in the config.
for draft_arch in ("DSparkDraftModel", "DeepSeekV4MTPModel"):
draft_cfg = DeepseekV4Config(architectures=[draft_arch], vision_n_layers=32)
conv = DeepseekV4ModelArchConfigConvertor(draft_cfg, draft_cfg)
assert conv.get_architectures() == [draft_arch]
assert draft_cfg.architectures == [draft_arch]
262 changes: 257 additions & 5 deletions tests/kernels/moe/test_topk_softplus_sqrt.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,46 @@ def _torch_topk_softplus_sqrt(
e_score_correction_bias: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
):
scores = F.softplus(gating_output.float()).sqrt()
original_scores = scores
if e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores

if hash_indices_table is not None:
assert input_ids is not None
topk_ids = hash_indices_table[input_ids.long()]
if bias_vl is None or image_sentinel_lo == 0:
topk_ids = hash_indices_table[input_ids.long()]
else:
image_mask = (input_ids.long() >= image_sentinel_lo) & (
input_ids.long() < image_sentinel_lo + 5
)
topk_ids = hash_indices_table[input_ids.long()]
vl_ids = (scores + bias_vl).topk(topk, dim=-1).indices
topk_ids = torch.where(
image_mask.unsqueeze(-1), vl_ids.to(topk_ids.dtype), topk_ids
)
else:
if bias_vl is not None and image_sentinel_lo > 0:
# Image tokens carry five consecutive in-vocab sentinel ids
# starting at image_sentinel_lo and select experts with bias_vl
# instead of the regular bias.
assert input_ids is not None
assert e_score_correction_bias is not None
image_mask = (
(input_ids.long() >= image_sentinel_lo)
& (input_ids.long() < image_sentinel_lo + 5)
).unsqueeze(-1)
row_bias = torch.where(
image_mask,
bias_vl.unsqueeze(0),
e_score_correction_bias.unsqueeze(0),
)
scores_for_choice = scores + row_bias
elif e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
topk_ids = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=True)[1]

topk_weights = original_scores.gather(1, topk_ids.long())
Expand Down Expand Up @@ -348,3 +376,227 @@ def test_fused_topk_softplus_sqrt_padding(
sorted_w_ref = topk_weights_ref[rows_to_compare].gather(1, idx_ref)
sorted_w = topk_weights[rows_to_compare].gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)


def _make_mixed_input_ids(
num_tokens: int, image_sentinel_lo: int, dtype: torch.dtype = torch.long
) -> torch.Tensor:
"""Mix of text, image-sentinel, and above-sentinel ids.

Every third token is an image sentinel id (cycling through the five slots
[image_sentinel_lo, image_sentinel_lo + 5)); every sixth token carries an
id just above the sentinel block, simulating the named special tokens
that follow it — those must route as regular text tokens.
"""
input_ids = torch.randint(
0, image_sentinel_lo, (num_tokens,), dtype=torch.long, device="cuda"
)
pos = torch.arange(num_tokens, device="cuda")
image_rows = pos % 3 == 0
input_ids[image_rows] = image_sentinel_lo + (pos[image_rows] // 3) % 5
above_rows = pos % 6 == 3
input_ids[above_rows] = image_sentinel_lo + 5
return input_ids.to(dtype)


def _assert_topk_matches(
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights_ref: torch.Tensor,
topk_ids_ref: torch.Tensor,
) -> None:
# Different kernels may return the topk experts in different orders when
# scores tie; sort by expert id before comparing.
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(
sorted_ref_ids, sorted_ids.to(sorted_ref_ids.dtype), atol=0, rtol=0
)

sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops.to(torch.long))
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)


@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype"),
[
(256, 6, True, torch.float32), # Triton dsv4 fast path
(256, 8, True, torch.bfloat16), # CUDA topk kernel
(384, 6, False, torch.half),
(512, 16, True, torch.float32),
],
)
def test_fused_topk_softplus_sqrt_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
):
"""Image sentinel tokens must select experts with bias_vl.

Rows with ids just above the sentinel block (image_sentinel_lo + 5)
simulate the named special tokens that follow it and must route through
the regular bias path; the reference comparison covers them.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 128
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
assert (input_ids == image_sentinel_lo + 5).any()

topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
e_score_correction_bias=e_score_correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)

topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=1.5,
input_tokens=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)

_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)


@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="This test is skipped on non-CUDA platform.",
)
@pytest.mark.parametrize(
("num_experts", "topk", "renormalize", "dtype", "indices_dtype"),
[
(256, 6, True, torch.float32, torch.long), # specialized hash kernel
(384, 6, True, torch.float32, torch.int32), # specialized hash kernel
(384, 8, False, torch.bfloat16, torch.long), # generic hash kernel
(512, 6, True, torch.float32, torch.long), # generic hash kernel
],
)
def test_fused_topk_softplus_sqrt_hash_bias_vl(
num_experts: int,
topk: int,
renormalize: bool,
dtype: torch.dtype,
indices_dtype: torch.dtype,
):
"""Hash MoE: text rows use tid2eid, image rows use topk(score + bias_vl).

Sentinel ids are in-vocab; rows with ids above the sentinel block
(image_sentinel_lo + 5) must still come straight from the hash table.
"""
torch.manual_seed(0)
num_tokens = 64
hidden_size = 1024
vocab_size = 64
image_sentinel_lo = vocab_size - 8
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=indices_dtype)
bias_vl = torch.randn((num_experts,), dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)
image_rows = (input_ids >= image_sentinel_lo) & (input_ids < image_sentinel_lo + 5)
assert image_rows.any() and (~image_rows).any()
assert (input_ids == image_sentinel_lo + 5).any()

topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=2.5,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)

topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=None,
topk=topk,
renormalize=renormalize,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
routed_scaling_factor=2.5,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)

_assert_topk_matches(topk_weights, topk_ids, topk_weights_ref, topk_ids_ref)

# Text rows must come straight from the hash table, image rows from
# topk(score + bias_vl).
text_ids = hash_indices_table[input_ids[~image_rows].long()]
torch.testing.assert_close(
topk_ids[~image_rows].to(text_ids.dtype), text_ids, atol=0, rtol=0
)


@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="The DeepSeek V4 fast path is CUDA-only.",
)
def test_dsv4_fast_topk_bias_vl():
torch.manual_seed(0)
num_tokens = 33
num_experts = 256
vocab_size = 128
image_sentinel_lo = vocab_size - 8
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.float32, device="cuda"
)
correction_bias = torch.randn(num_experts, dtype=torch.float32, device="cuda")
bias_vl = torch.randn(num_experts, dtype=torch.float32, device="cuda")
input_ids = _make_mixed_input_ids(num_tokens, image_sentinel_lo)

topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=6,
renormalize=True,
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)
topk_weights, topk_ids = dsv4_topk(
gating_output,
correction_bias,
torch.int64,
1.5,
input_ids=input_ids,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
)

assert topk_ids.dtype == torch.int64
torch.testing.assert_close(topk_ids_ref.to(torch.int64), topk_ids, atol=0, rtol=0)
torch.testing.assert_close(topk_weights_ref, topk_weights, atol=2e-5, rtol=2e-5)
4 changes: 4 additions & 0 deletions tests/models/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -832,6 +832,10 @@ def check_available_online(
"DeepseekOCR2ForCausalLM": _HfExamplesInfo(
"deepseek-ai/DeepSeek-OCR-2",
),
"DeepseekV4ForConditionalGeneration": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
is_available_online=False,
),
"Dots3NoteForCausalLM": _HfExamplesInfo(
"dots-studio/dots3-note-prev",
is_available_online=False,
Expand Down
Loading