Skip to content
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions python/sglang/srt/layers/hc_mix_flydsl.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
"""Optional gfx950 HC mix using AITER's FlyDSL fused GEMM epilogues."""

from functools import lru_cache
from importlib.util import find_spec

import torch

# AITER's kernel limit; it stays ahead of the torch.compile fallback up to it
# (K=10240, R=320, cold weights: 26 vs 42 us at 128 rows).
_MAX_ROWS = 128


@lru_cache(maxsize=1)
def _aiter_hc_mix_available():
# Check only after device/shape guards; optional packages stay lazy.
try:
return (
find_spec("flydsl") is not None
and find_spec("aiter.ops.flydsl.hc_mix") is not None
)
except ModuleNotFoundError:
return False


def flydsl_hc_mix_weights_supported(down, up, hc):
"""Device and weight-shape half of the check.

None of it changes after the weights are loaded, so callers evaluate it
once per layer and pass the result to ``flydsl_hc_mix_supported``.
"""
# Keep FlyDSL/AITER imports off the NVIDIA path.
if torch.version.hip is None:
return False
r, k = down.shape
# pack_hc_weights tiles K by 64 and R by 32, and makes its own contiguous
# copies, so the source weights' layout does not matter.
return (
hc == 4
and r > 0
and r % 32 == 0
and k > 0
and k % 64 == 0
and up.shape == (k, r)
and down.dtype == up.dtype == torch.bfloat16
and down.is_cuda
and up.device == down.device
and torch.cuda.get_device_properties(down.device).gcnArchName.split(":")[0]
== "gfx950"
and _aiter_hc_mix_available()
)


def flydsl_hc_mix_supported(x, weights_supported):
"""Per-call half: the activation's dtype, row count and layout.

``x`` is the [rows, HC*HS] normed input, which the caller has already
matched to the weights' K.
"""
return (
weights_supported
and x.dtype == torch.bfloat16
and x.ndim == 2
and 1 <= x.shape[0] <= _MAX_ROWS
and x.is_contiguous()
and x.data_ptr() % 16 == 0
)


def weight_cache_key(down, up):
# Inference tensors do not have version counters. As in the CuTe weight
# cache, in-place updates of those tensors require reinitializing the cache.
def version(w):
return None if w.is_inference() else w._version

return (down.data_ptr(), version(down), up.data_ptr(), version(up))
38 changes: 37 additions & 1 deletion python/sglang/srt/layers/hyperconnection.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@
import torch.nn.functional as F

from sglang.kernels.ops.gemm.hc_mix import fused_hc_mix, fused_hc_mix_supported
from sglang.srt.layers.hc_mix_flydsl import (
flydsl_hc_mix_supported,
flydsl_hc_mix_weights_supported,
weight_cache_key,
)
from sglang.srt.utils import is_hip

_is_hip = is_hip()


@lru_cache(None)
Expand Down Expand Up @@ -164,6 +172,10 @@ def __init__(
and lowrank % 8 == 0
)
self._mix_up_weight_padded = None
# Resolved on the first mix(), once the weights are final.
self._mix_flydsl_ok = None
self._mix_flydsl_weights = None
self._mix_flydsl_weight_key = None

self._gate_stream = None
if use_combine:
Expand All @@ -179,8 +191,11 @@ def __init__(
and (self.hc_count * self.hidden_size) % 2048 == 0
)
vecs = self.hc_count * self.hidden_size // 8
# Off on ROCm/HIP: on gfx950 Qwen decode both halves of the split pair
# sit at the ~4.2us launch floor, slower than the single fused kernel.
self._split_combine_ok = (
self._jit_combine_ok
not _is_hip
Comment thread
amd-nprotaso marked this conversation as resolved.
Resolved
and self._jit_combine_ok
and vecs % (8 * 160) == 0
and (self.hidden_size // 8) % (vecs // 8) == 0
)
Expand Down Expand Up @@ -233,6 +248,15 @@ def _combine_compute(
self._mix_compute = torch.compile(_mix_compute)
self._combine_compute = torch.compile(_combine_compute)

def _flydsl_mix_ok(self) -> bool:
if self._mix_flydsl_ok is None:
self._mix_flydsl_ok = flydsl_hc_mix_weights_supported(
self.input_mix_weight_down.weight,
self.input_mix_weight_up.weight,
self.hc_count,
)
return self._mix_flydsl_ok

def mix(
self,
hyper_input: torch.Tensor,
Expand Down Expand Up @@ -277,6 +301,18 @@ def mix(
self.hc_count,
self.hidden_size,
).to(self.params_dtype)
elif flydsl_hc_mix_supported(hyper_input_normed, self._flydsl_mix_ok()):
from aiter.ops.flydsl.hc_mix import hc_mix, pack_hc_weights

down = self.input_mix_weight_down.weight
up = self.input_mix_weight_up.weight
key = weight_cache_key(down, up)
if self._mix_flydsl_weight_key != key:
self._mix_flydsl_weights = pack_hc_weights(down, up, self.hc_count)
self._mix_flydsl_weight_key = key
mixed_input = hc_mix(hyper_input_normed, *self._mix_flydsl_weights).to(
self.params_dtype
)
elif fused_hc_mix_supported(
hyper_input_normed,
self.input_mix_weight_down.weight,
Expand Down
105 changes: 105 additions & 0 deletions test/registered/kernels/ops/gemm/test_hc_mix_flydsl.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import sys
from unittest.mock import patch

import pytest
import torch

from sglang.srt.layers.hc_mix_flydsl import (
flydsl_hc_mix_supported,
flydsl_hc_mix_weights_supported,
)
from sglang.test.ci.ci_register import register_amd_ci

register_amd_ci(est_time=30, suite="stage-b-test-1-gpu-small-amd")

pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or torch.version.hip is None,
reason="Requires ROCm gfx950",
)


def test_gated_residual_dispatch():
if torch.cuda.get_device_properties().gcnArchName.split(":")[0] != "gfx950":
pytest.skip("Requires gfx950")
from aiter.ops.flydsl.hc_mix import hc_mix

from sglang.srt.layers.hyperconnection import GatedResidual, HyperConnectionConfig

cfg = HyperConnectionConfig(hidden_size=2560, hc_lowrank=320)
layer = GatedResidual(cfg, use_combine=False).cuda().bfloat16()
x = torch.randn(4, 10240, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
with patch("aiter.ops.flydsl.hc_mix.hc_mix", wraps=hc_mix) as call:
result, (_, normed, _) = layer.mix(x)
assert call.call_count == 1
d, u = layer.input_mix_weight_down.weight, layer.input_mix_weight_up.weight
assert layer._mix_flydsl_ok
with patch(
"sglang.srt.layers.hc_mix_flydsl._aiter_hc_mix_available",
return_value=False,
):
assert not flydsl_hc_mix_weights_supported(d, u, 4)
t = torch.nn.functional.silu(normed.float() @ d.float().T / 4).bfloat16()
ref = (
((t.float() @ u.float().T).sigmoid() * normed.float())
.view(4, 4, 2560)
.mean(1)
.bfloat16()
)
torch.testing.assert_close(result, ref, atol=0.004, rtol=0.01)
packed = layer._mix_flydsl_weights
layer.mix(x)
assert layer._mix_flydsl_weights is packed
d.mul_(0.5)
layer.mix(x)
assert layer._mix_flydsl_weights is not packed
assert flydsl_hc_mix_supported(x.repeat(32, 1), True)
for bad_x in [x[:0], x.repeat(33, 1), x.half(), x[:, ::2]]:
assert not flydsl_hc_mix_supported(bad_x, True)
assert not flydsl_hc_mix_supported(x, False)
assert not flydsl_hc_mix_weights_supported(d, u, 5)
# K only needs whole 64-wide tiles, not the old 2048 multiple.
k = 64 * 33
assert flydsl_hc_mix_weights_supported(d[:, :k], u[:k], 4)
assert not flydsl_hc_mix_weights_supported(d[:, : k - 32], u[: k - 32], 4)
assert not flydsl_hc_mix_weights_supported(d[:16], u[:, :16], 4)
assert not flydsl_hc_mix_weights_supported(d.half(), u.half(), 4)


@pytest.mark.parametrize("rows", [32, 96, 128])
def test_gated_residual_dispatch_many_rows(rows):
# Speculative verify passes carry requests x draft tokens rows.
if torch.cuda.get_device_properties().gcnArchName.split(":")[0] != "gfx950":
pytest.skip("Requires gfx950")
from aiter.ops.flydsl.hc_mix import hc_mix

from sglang.srt.layers.hyperconnection import GatedResidual, HyperConnectionConfig

cfg = HyperConnectionConfig(hidden_size=2560, hc_lowrank=320)
layer = GatedResidual(cfg, use_combine=False).cuda().bfloat16()
x = torch.randn(rows, 10240, device="cuda", dtype=torch.bfloat16)
with torch.no_grad():
with patch("aiter.ops.flydsl.hc_mix.hc_mix", wraps=hc_mix) as call:
result, (_, normed, _) = layer.mix(x)
assert call.call_count == 1
d, u = layer.input_mix_weight_down.weight, layer.input_mix_weight_up.weight
t = torch.nn.functional.silu(normed.float() @ d.float().T / 4).bfloat16()
ref = (
((t.float() @ u.float().T).sigmoid() * normed.float())
.view(rows, 4, 2560)
.mean(1)
.bfloat16()
)
torch.testing.assert_close(result, ref, atol=0.004, rtol=0.01)


def test_inference_tensor_weight_cache():
from sglang.srt.layers.hc_mix_flydsl import weight_cache_key

with torch.inference_mode():
w = torch.ones(4)
assert weight_cache_key(w, w)[1] is None


if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
Loading