Repository navigation
[Perf] [AMD] Qwen3.8: AITER FlyDSL hc_mix for Qwen3.8 hyper-connections on gfx950 #41880
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 12 commits
Commits
Show all changes
15 commits
Select commit
Hold shift + click to select a range
bb4f7d7
add hc mix kernel
amd-nprotaso d1758f2
Apply isort and ruff-format to new FlyDSL hc_mix files
amd-nprotaso d1e676e
add env varable to use spllit combine hc kernel
amd-nprotaso 17efd0b
Default SGLANG_HC_COMBINE_SPLIT off on ROCm/HIP
amd-nprotaso 7f54a2f
fix hc kernels
amd-nprotaso d98023d
remove env flags guard
amd-nprotaso c24fb69
remove kernel benchmark
amd-nprotaso 0bc1153
Merge branch 'main' into pr/hc-mix-flydsl
amd-nprotaso 8d8a645
Merge branch 'main' of https://github.com/sgl-project/sglang into pr/…
amd-nprotaso 1c57400
Merge branch 'pr/hc-mix-flydsl' of https://github.com/amd-nprotaso/sg…
amd-nprotaso b1cda98
Merge branch 'main' into pr/hc-mix-flydsl
amd-nprotaso 25ac33a
Merge branch 'main' into pr/hc-mix-flydsl
1am9trash 80fd440
stay previous logic of hc_combine_split call
amd-nprotaso 3e7a24d
Merge branch 'main' into pr/hc-mix-flydsl
amd-nprotaso c06b2f5
fix lint
amd-nprotaso File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| """Optional gfx950 HC mix using AITER's FlyDSL fused GEMM epilogues.""" | ||
|
|
||
| from functools import lru_cache | ||
| from importlib.util import find_spec | ||
|
|
||
| import torch | ||
|
|
||
| # AITER's kernel limit; it stays ahead of the torch.compile fallback up to it | ||
| # (K=10240, R=320, cold weights: 26 vs 42 us at 128 rows). | ||
| _MAX_ROWS = 128 | ||
|
|
||
|
|
||
| @lru_cache(maxsize=1) | ||
| def _aiter_hc_mix_available(): | ||
| # Check only after device/shape guards; optional packages stay lazy. | ||
| try: | ||
| return ( | ||
| find_spec("flydsl") is not None | ||
| and find_spec("aiter.ops.flydsl.hc_mix") is not None | ||
| ) | ||
| except ModuleNotFoundError: | ||
| return False | ||
|
|
||
|
|
||
| def flydsl_hc_mix_weights_supported(down, up, hc): | ||
| """Device and weight-shape half of the check. | ||
|
|
||
| None of it changes after the weights are loaded, so callers evaluate it | ||
| once per layer and pass the result to ``flydsl_hc_mix_supported``. | ||
| """ | ||
| # Keep FlyDSL/AITER imports off the NVIDIA path. | ||
| if torch.version.hip is None: | ||
| return False | ||
| r, k = down.shape | ||
| # pack_hc_weights tiles K by 64 and R by 32, and makes its own contiguous | ||
| # copies, so the source weights' layout does not matter. | ||
| return ( | ||
| hc == 4 | ||
| and r > 0 | ||
| and r % 32 == 0 | ||
| and k > 0 | ||
| and k % 64 == 0 | ||
| and up.shape == (k, r) | ||
| and down.dtype == up.dtype == torch.bfloat16 | ||
| and down.is_cuda | ||
| and up.device == down.device | ||
| and torch.cuda.get_device_properties(down.device).gcnArchName.split(":")[0] | ||
| == "gfx950" | ||
| and _aiter_hc_mix_available() | ||
| ) | ||
|
|
||
|
|
||
| def flydsl_hc_mix_supported(x, weights_supported): | ||
| """Per-call half: the activation's dtype, row count and layout. | ||
|
|
||
| ``x`` is the [rows, HC*HS] normed input, which the caller has already | ||
| matched to the weights' K. | ||
| """ | ||
| return ( | ||
| weights_supported | ||
| and x.dtype == torch.bfloat16 | ||
| and x.ndim == 2 | ||
| and 1 <= x.shape[0] <= _MAX_ROWS | ||
| and x.is_contiguous() | ||
| and x.data_ptr() % 16 == 0 | ||
| ) | ||
|
|
||
|
|
||
| def weight_cache_key(down, up): | ||
| # Inference tensors do not have version counters. As in the CuTe weight | ||
| # cache, in-place updates of those tensors require reinitializing the cache. | ||
| def version(w): | ||
| return None if w.is_inference() else w._version | ||
|
|
||
| return (down.data_ptr(), version(down), up.data_ptr(), version(up)) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| import sys | ||
| from unittest.mock import patch | ||
|
|
||
| import pytest | ||
| import torch | ||
|
|
||
| from sglang.srt.layers.hc_mix_flydsl import ( | ||
| flydsl_hc_mix_supported, | ||
| flydsl_hc_mix_weights_supported, | ||
| ) | ||
| from sglang.test.ci.ci_register import register_amd_ci | ||
|
|
||
| register_amd_ci(est_time=30, suite="stage-b-test-1-gpu-small-amd") | ||
|
|
||
| pytestmark = pytest.mark.skipif( | ||
| not torch.cuda.is_available() or torch.version.hip is None, | ||
| reason="Requires ROCm gfx950", | ||
| ) | ||
|
|
||
|
|
||
| def test_gated_residual_dispatch(): | ||
| if torch.cuda.get_device_properties().gcnArchName.split(":")[0] != "gfx950": | ||
| pytest.skip("Requires gfx950") | ||
| from aiter.ops.flydsl.hc_mix import hc_mix | ||
|
|
||
| from sglang.srt.layers.hyperconnection import GatedResidual, HyperConnectionConfig | ||
|
|
||
| cfg = HyperConnectionConfig(hidden_size=2560, hc_lowrank=320) | ||
| layer = GatedResidual(cfg, use_combine=False).cuda().bfloat16() | ||
| x = torch.randn(4, 10240, device="cuda", dtype=torch.bfloat16) | ||
| with torch.no_grad(): | ||
| with patch("aiter.ops.flydsl.hc_mix.hc_mix", wraps=hc_mix) as call: | ||
| result, (_, normed, _) = layer.mix(x) | ||
| assert call.call_count == 1 | ||
| d, u = layer.input_mix_weight_down.weight, layer.input_mix_weight_up.weight | ||
| assert layer._mix_flydsl_ok | ||
| with patch( | ||
| "sglang.srt.layers.hc_mix_flydsl._aiter_hc_mix_available", | ||
| return_value=False, | ||
| ): | ||
| assert not flydsl_hc_mix_weights_supported(d, u, 4) | ||
| t = torch.nn.functional.silu(normed.float() @ d.float().T / 4).bfloat16() | ||
| ref = ( | ||
| ((t.float() @ u.float().T).sigmoid() * normed.float()) | ||
| .view(4, 4, 2560) | ||
| .mean(1) | ||
| .bfloat16() | ||
| ) | ||
| torch.testing.assert_close(result, ref, atol=0.004, rtol=0.01) | ||
| packed = layer._mix_flydsl_weights | ||
| layer.mix(x) | ||
| assert layer._mix_flydsl_weights is packed | ||
| d.mul_(0.5) | ||
| layer.mix(x) | ||
| assert layer._mix_flydsl_weights is not packed | ||
| assert flydsl_hc_mix_supported(x.repeat(32, 1), True) | ||
| for bad_x in [x[:0], x.repeat(33, 1), x.half(), x[:, ::2]]: | ||
| assert not flydsl_hc_mix_supported(bad_x, True) | ||
| assert not flydsl_hc_mix_supported(x, False) | ||
| assert not flydsl_hc_mix_weights_supported(d, u, 5) | ||
| # K only needs whole 64-wide tiles, not the old 2048 multiple. | ||
| k = 64 * 33 | ||
| assert flydsl_hc_mix_weights_supported(d[:, :k], u[:k], 4) | ||
| assert not flydsl_hc_mix_weights_supported(d[:, : k - 32], u[: k - 32], 4) | ||
| assert not flydsl_hc_mix_weights_supported(d[:16], u[:, :16], 4) | ||
| assert not flydsl_hc_mix_weights_supported(d.half(), u.half(), 4) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("rows", [32, 96, 128]) | ||
| def test_gated_residual_dispatch_many_rows(rows): | ||
| # Speculative verify passes carry requests x draft tokens rows. | ||
| if torch.cuda.get_device_properties().gcnArchName.split(":")[0] != "gfx950": | ||
| pytest.skip("Requires gfx950") | ||
| from aiter.ops.flydsl.hc_mix import hc_mix | ||
|
|
||
| from sglang.srt.layers.hyperconnection import GatedResidual, HyperConnectionConfig | ||
|
|
||
| cfg = HyperConnectionConfig(hidden_size=2560, hc_lowrank=320) | ||
| layer = GatedResidual(cfg, use_combine=False).cuda().bfloat16() | ||
| x = torch.randn(rows, 10240, device="cuda", dtype=torch.bfloat16) | ||
| with torch.no_grad(): | ||
| with patch("aiter.ops.flydsl.hc_mix.hc_mix", wraps=hc_mix) as call: | ||
| result, (_, normed, _) = layer.mix(x) | ||
| assert call.call_count == 1 | ||
| d, u = layer.input_mix_weight_down.weight, layer.input_mix_weight_up.weight | ||
| t = torch.nn.functional.silu(normed.float() @ d.float().T / 4).bfloat16() | ||
| ref = ( | ||
| ((t.float() @ u.float().T).sigmoid() * normed.float()) | ||
| .view(rows, 4, 2560) | ||
| .mean(1) | ||
| .bfloat16() | ||
| ) | ||
| torch.testing.assert_close(result, ref, atol=0.004, rtol=0.01) | ||
|
|
||
|
|
||
| def test_inference_tensor_weight_cache(): | ||
| from sglang.srt.layers.hc_mix_flydsl import weight_cache_key | ||
|
|
||
| with torch.inference_mode(): | ||
| w = torch.ones(4) | ||
| assert weight_cache_key(w, w)[1] is None | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| sys.exit(pytest.main([__file__, "-v"])) |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.