Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 38 additions & 46 deletions tests/kernels/moe/test_rocm_aiter_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -565,72 +565,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected


def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
@pytest.mark.parametrize(
"value,expected_act,expected_env",
[
# Unset / auto / legacy "1" all mean the a4w4 default.
(None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("0", "a16w4", {}),
("a16w4", "a16w4", {}),
("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
],
)
def test_aiter_moe_situv2_activation_syncs_aiter_env(
value: str | None,
expected_act: str,
expected_env: dict[str, str],
monkeypatch: pytest.MonkeyPatch,
):
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
from vllm._aiter_ops import rocm_aiter_ops

_assert_aiter_supported()

with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()

import os

assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ

"""Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
import os

def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
monkeypatch: pytest.MonkeyPatch,
):
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
from vllm._aiter_ops import rocm_aiter_ops

_assert_aiter_supported()

with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
# Stale AITER overrides must be replaced by the vLLM selection.
mp.setenv("AITER_SITUV2_A8W4", "1")
mp.setenv("AITER_SITUV2_A4W4", "1")
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
if value is None:
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
else:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
_reload_envs()
rocm_aiter_ops.refresh_env_variables()

import os

assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
expected_act == "a8w4"
)
assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
assert os.environ.get(name) == expected_env.get(name)


def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
def test_aiter_moe_situv2_rejects_unknown_activation(
monkeypatch: pytest.MonkeyPatch,
):
from vllm._aiter_ops import rocm_aiter_ops

_assert_aiter_supported()

with monkeypatch.context() as mp:
mp.setenv("AITER_SITUV2_A4W4", "1")
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()

import os
import vllm.envs as envs

assert "AITER_SITUV2_A4W4" not in os.environ
with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
_ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2


@pytest.mark.parametrize("moe_padding", [True, False])
Expand Down
52 changes: 38 additions & 14 deletions vllm/_aiter_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -1870,13 +1870,19 @@ def _mhc_delayed_pre_tail(
_OPS_REGISTERED = False


_AITER_SITUV2_ACT_ENV = {
"a8w4": "AITER_SITUV2_A8W4",
"a4w4": "AITER_SITUV2_A4W4",
}


def _sync_aiter_situv2_moe_env() -> None:
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
"""Mirror VLLM_ROCM_USE_AITER_MOE_SITUV2 into AITER's SiTUv2 dispatch env.

AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
AITER picks the SiTUv2 activation dtype from AITER_SITUV2_A8W4 /
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first) and
defaults to bf16 (a16w4). Exactly one of the two is set for a8w4/a4w4
and both are cleared for a16w4, so the vLLM option is authoritative.

Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
Expand All @@ -1887,11 +1893,12 @@ def _sync_aiter_situv2_moe_env() -> None:

import vllm.envs as envs

if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
os.environ["AITER_SITUV2_A4W4"] = "1"
os.environ.pop("AITER_SITUV2_A8W4", None)
else:
os.environ.pop("AITER_SITUV2_A4W4", None)
selected = _AITER_SITUV2_ACT_ENV.get(envs.VLLM_ROCM_USE_AITER_MOE_SITUV2)
for name in _AITER_SITUV2_ACT_ENV.values():
if name == selected:
os.environ[name] = "1"
else:
os.environ.pop(name, None)


class rocm_aiter_ops:
Expand All @@ -1917,7 +1924,8 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
dtype (a16w4 | a8w4 | a4w4).
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.

Note:
Expand Down Expand Up @@ -2131,9 +2139,25 @@ def is_fusion_moe_shared_experts_enabled(cls) -> bool:
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_enabled(cls) -> bool:
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
# fused moe must be enabled as well.
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
"""True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"

@classmethod
@if_aiter_supported
def get_fused_moe_situv2_activation(cls) -> str:
"""SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
return cls._MOE_SITUV2

@classmethod
@if_aiter_supported
def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
"""Whether the SiTUv2 kernels expect gate/up-interleaved w13.

The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
(experts/rocm_aiter_moe.py) must both use this.
"""
return cls._MOE_SITUV2 == "a8w4"

@classmethod
@if_aiter_supported
Expand Down
47 changes: 35 additions & 12 deletions vllm/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
VLLM_ROCM_USE_AITER_MOE: bool = True
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["a16w4", "a8w4", "a4w4"] = "a4w4"
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
Expand Down Expand Up @@ -433,6 +433,31 @@ def _get_validated_env() -> str | None:
return _get_validated_env


_AITER_MOE_SITUV2_CHOICES = ("a16w4", "a8w4", "a4w4")
_AITER_MOE_SITUV2_AUTO = "a4w4"


def _aiter_moe_situv2_activation() -> str:
"""Resolve VLLM_ROCM_USE_AITER_MOE_SITUV2 to a SiTUv2 activation dtype.

Unset, empty or "auto" picks the default (a4w4). Accepts the
a16w4/a8w4/a4w4 names and the legacy boolean form (1 -> a4w4,
0 -> a16w4).
"""
value = os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2")
norm = (value or "").strip().lower()
if norm in ("", "auto", "true", "1"):
return _AITER_MOE_SITUV2_AUTO
if norm in ("false", "0"):
return "a16w4"
if norm not in _AITER_MOE_SITUV2_CHOICES:
raise ValueError(
f"Invalid value '{value}' for VLLM_ROCM_USE_AITER_MOE_SITUV2. "
f"Valid options: {list(_AITER_MOE_SITUV2_CHOICES)}, auto, 0 or 1."
)
return norm


def env_list_with_choices(
env_name: str,
default: list[str],
Expand Down Expand Up @@ -1269,19 +1294,17 @@ def _resolve_rust_cli_path() -> str | None:
"VLLM_ROCM_USE_AITER_MOE": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
),
# Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
# activations, separated gate/up layout) instead of default a16w4. vLLM
# sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
# legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
# VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
# recipes; it does not select a8w4 kernels.
# Activation dtype for the K3 SiTU MXFP4 MoE on the FlyDSL SiTUv2 path.
# Each value selects a different FlyDSL kernel family and tuned config
# (kimik3_{a16w4,a8w4,a4w4}_tuned_fmoe.csv):
# a4w4 (default / "auto"): fp4 activations, fastest (AITER_SITUV2_A4W4=1)
# a8w4: fp8 activations, gate/up interleaved (AITER_SITUV2_A8W4=1)
# a16w4: bf16 activations, most accurate
# vLLM mirrors the choice into the AITER_SITUV2_* env at init. "1"/"0"
# keep their old meaning (a4w4 / a16w4).
# Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
# and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
"VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
in ("true", "1")
),
"VLLM_ROCM_USE_AITER_MOE_SITUV2": _aiter_moe_situv2_activation,
# MoE sorting dispatch policy for AITER fused MoE kernels.
# 0 = auto (default): single-pass for small batches, multi-pass
# for large batches
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -385,10 +385,11 @@ def rocm_aiter_fused_experts(

gate_mode = ""
if activation == MoEActivation.SITU:
# SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
# fp4 activations with separated gate/up weights (AITER #4463);
# default a16w4 SiTU also stays separated.
gate_mode = GateMode.SEPARATED.value
gate_mode = (
GateMode.INTERLEAVE.value
if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
else GateMode.SEPARATED.value
)
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
Expand Down
3 changes: 1 addition & 2 deletions vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
Original file line number Diff line number Diff line change
Expand Up @@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(

fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# SiTUv2 flydsl uses separated gate/up layout (a4w4).
guinterleave = False
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
w13_weight.data.view(fp4_dtype), 16, guinterleave
)
Expand Down
Loading