From d531d95bf57fa84dd6b5a5cb7a58fc649e3cdac8 Mon Sep 17 00:00:00 2001 From: alphabetc1 <2508695655@qq.com> Date: Wed, 2 Sep 2026 18:41:01 +0800 Subject: [PATCH 1/4] fix: boot Inkling on Hopper without an explicit attention backend --- .../sglang/srt/arg_groups/cuda_graph_hook.py | 20 +++++++++++++------ .../srt/arg_groups/model_overrides/inkling.py | 14 ++++++++++++- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 2c325b6c3dcc..4fe47af9bde7 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -323,15 +323,23 @@ def apply_inkling_prefill_cuda_graph_default(server_args: Any): ): return arch = model_config_of(server_args).hf_config.architectures[0] - if arch in ( + if arch not in ( "InklingForConditionalGeneration", "InklingForConditionalGenerationMTP", ): - declare_resolution( - server_args, - "_apply_inkling_prefill_cuda_graph_default", - cuda_graph_backend_prefill=Backend.FULL, - ) + return + # Triton cannot capture EXTEND; only fa4 opts into a full prefill graph. + from sglang.srt.arg_groups.model_overrides.inkling import ( + resolve_inkling_attention_backend, + ) + + if resolve_inkling_attention_backend(server_args) != "fa4": + return + declare_resolution( + server_args, + "_apply_inkling_prefill_cuda_graph_default", + cuda_graph_backend_prefill=Backend.FULL, + ) def apply_muse_glimmer_prefill_cuda_graph_max_bs_default(server_args: Any): diff --git a/python/sglang/srt/arg_groups/model_overrides/inkling.py b/python/sglang/srt/arg_groups/model_overrides/inkling.py index bfc7caf1b394..347522b4b731 100644 --- a/python/sglang/srt/arg_groups/model_overrides/inkling.py +++ b/python/sglang/srt/arg_groups/model_overrides/inkling.py @@ -13,6 +13,18 @@ logger = logging.getLogger(__name__) +def resolve_inkling_attention_backend(server_args: Any) -> str: + """Also read by the prefill-graph default, before overrides materialize.""" + cfg = resolving_view(server_args) + if not is_attention_backend_not_set(cfg): + return ( + cfg.attention_backend + or cfg.prefill_attention_backend + or cfg.decode_attention_backend + ) + return "fa4" if get_platform().is_sm100 else "triton" + + @_register_for( "InklingForConditionalGeneration", "InklingForConditionalGenerationMTP", @@ -66,7 +78,7 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict: # (mirrors the MiniMax-M3 SM100 fa4-default above); an explicit # --attention-backend / --prefill/decode-attention-backend still wins. if is_attention_backend_not_set(cfg): - inkling_attn_backend = "fa4" if get_platform().is_sm100 else "triton" + inkling_attn_backend = resolve_inkling_attention_backend(server_args) overrides["attention_backend"] = inkling_attn_backend logger.info( f"Use {inkling_attn_backend} as the attention backend for Inkling " From 7932f3d56303083b097ed250c9148ceb31755e9e Mon Sep 17 00:00:00 2001 From: alphabetc1 <2508695655@qq.com> Date: Sun, 11 Oct 2026 11:05:56 +0800 Subject: [PATCH 2/4] fix: default Inkling to fa4 on Hopper and key the full prefill graph on the prefill backend --- .../srt/arg_groups/model_overrides/inkling.py | 17 +++----- .../test_multimodal_piecewise_cuda_graph.py | 43 ++++++++++++++++++- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/python/sglang/srt/arg_groups/model_overrides/inkling.py b/python/sglang/srt/arg_groups/model_overrides/inkling.py index 347522b4b731..c511b55d9084 100644 --- a/python/sglang/srt/arg_groups/model_overrides/inkling.py +++ b/python/sglang/srt/arg_groups/model_overrides/inkling.py @@ -1,7 +1,7 @@ """Config-time override declarations for inkling.""" import logging -from typing import Any, Dict +from typing import Any, Dict, Optional from sglang.srt.arg_groups.model_override_base import ( _register_for, @@ -13,16 +13,13 @@ logger = logging.getLogger(__name__) -def resolve_inkling_attention_backend(server_args: Any) -> str: - """Also read by the prefill-graph default, before overrides materialize.""" +def resolve_inkling_attention_backend(server_args: Any) -> Optional[str]: + """The backend Inkling prefill runs on; the prefill-graph default reads it + before overrides materialize.""" cfg = resolving_view(server_args) if not is_attention_backend_not_set(cfg): - return ( - cfg.attention_backend - or cfg.prefill_attention_backend - or cfg.decode_attention_backend - ) - return "fa4" if get_platform().is_sm100 else "triton" + return cfg.prefill_attention_backend or cfg.attention_backend + return "fa4" if get_platform().is_sm90 or get_platform().is_sm100 else "triton" @_register_for( @@ -70,7 +67,7 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict: # handle_mamba_radix_cache rejects extra_buffer on a model without it. if not cfg.disable_radix_cache: overrides["uses_mamba_radix_cache"] = True - # Inkling attention runs only on the fa4 (Blackwell) or triton backends -- + # Inkling attention runs only on the fa4 (Hopper/Blackwell) or triton backends -- # models/inkling_common/attn.py asserts attention_backend in {fa4, triton}. # The generic resolver would otherwise pick trtllm_mha (SM100) / fa3 # (Hopper), so a bare launch fails on the first attention forward. Pin a diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index bc2324999e3e..e8d60865d35e 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -4,8 +4,12 @@ from types import SimpleNamespace from unittest.mock import patch -from sglang.srt.arg_groups.cuda_graph_hook import apply_cuda_graph_compatibility +from sglang.srt.arg_groups.cuda_graph_hook import ( + apply_cuda_graph_compatibility, + apply_inkling_prefill_cuda_graph_default, +) from sglang.srt.arg_groups.model_hook import handle_model_capability_adjustments +from sglang.srt.arg_groups.model_overrides.inkling import _inkling_overrides from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec from sglang.srt.configs.model_config import ( @@ -103,6 +107,43 @@ def test_breakable_prefill_takes_nonzero_prefix_on_cuda_only(self): with patch(target, return_value=False): self.assertFalse(runner.can_run_graph(forward_batch)) + def test_inkling_full_prefill_graph_follows_the_attention_backend(self): + # (platform, explicit backend flags) -> (attention backend, prefill graph) + cases = [ + (dict(is_sm90=True, is_sm100=False), {}, "fa4", Backend.FULL), + (dict(is_sm90=False, is_sm100=True), {}, "fa4", Backend.FULL), + (dict(is_sm90=False, is_sm100=False), {}, "triton", None), + ( + dict(is_sm90=True, is_sm100=False), + dict(attention_backend="triton"), + None, + None, + ), + ( + dict(is_sm90=True, is_sm100=False), + dict(attention_backend="fa4", prefill_attention_backend="triton"), + None, + None, + ), + ] + for platform, flags, attention_backend, prefill_backend in cases: + with self.subTest(platform=platform, flags=flags): + args = ServerArgs(model_path="dummy", **flags) + args._model_config = SimpleNamespace( + hf_config=SimpleNamespace( + architectures=["InklingForConditionalGeneration"] + ), + ) + with override_platform(is_cuda=True, **platform): + overrides = _inkling_overrides(args, None) + apply_inkling_prefill_cuda_graph_default(args) + + self.assertEqual(overrides.get("attention_backend"), attention_backend) + self.assertEqual( + resolution_result(args, "cuda_graph_backend_prefill"), + prefill_backend, + ) + def test_embedding_gemma_forces_breakable_prefill(self): args = ServerArgs(model_path="dummy") args._model_config = SimpleNamespace( From 9400a26a581e50a09edc2fc3bdf95f5f08e3cdb2 Mon Sep 17 00:00:00 2001 From: alphabetc1 <2508695655@qq.com> Date: Sun, 11 Oct 2026 11:26:04 +0800 Subject: [PATCH 3/4] refactor: inline the Inkling Hopper fa4 default and drop the unit test --- .../sglang/srt/arg_groups/cuda_graph_hook.py | 14 +++--- .../srt/arg_groups/model_overrides/inkling.py | 16 +++---- .../test_multimodal_piecewise_cuda_graph.py | 43 +------------------ 3 files changed, 14 insertions(+), 59 deletions(-) diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 4fe47af9bde7..4fb8b8879582 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -22,6 +22,7 @@ default_cuda_graph_config, with_phase, ) +from sglang.srt.runtime_context import get_platform from sglang.srt.utils.common import ( parse_connector_type, ) @@ -328,12 +329,13 @@ def apply_inkling_prefill_cuda_graph_default(server_args: Any): "InklingForConditionalGenerationMTP", ): return - # Triton cannot capture EXTEND; only fa4 opts into a full prefill graph. - from sglang.srt.arg_groups.model_overrides.inkling import ( - resolve_inkling_attention_backend, - ) - - if resolve_inkling_attention_backend(server_args) != "fa4": + # Only fa4 captures EXTEND; an unset backend defaults to fa4 on SM90/SM100. + prefill_backend, _ = attention_backends_of(cfg) + if prefill_backend is None: + platform = get_platform() + if not (platform.is_sm90 or platform.is_sm100): + return + elif prefill_backend != "fa4": return declare_resolution( server_args, diff --git a/python/sglang/srt/arg_groups/model_overrides/inkling.py b/python/sglang/srt/arg_groups/model_overrides/inkling.py index c511b55d9084..12f63eecb1c3 100644 --- a/python/sglang/srt/arg_groups/model_overrides/inkling.py +++ b/python/sglang/srt/arg_groups/model_overrides/inkling.py @@ -1,7 +1,7 @@ """Config-time override declarations for inkling.""" import logging -from typing import Any, Dict, Optional +from typing import Any, Dict from sglang.srt.arg_groups.model_override_base import ( _register_for, @@ -13,15 +13,6 @@ logger = logging.getLogger(__name__) -def resolve_inkling_attention_backend(server_args: Any) -> Optional[str]: - """The backend Inkling prefill runs on; the prefill-graph default reads it - before overrides materialize.""" - cfg = resolving_view(server_args) - if not is_attention_backend_not_set(cfg): - return cfg.prefill_attention_backend or cfg.attention_backend - return "fa4" if get_platform().is_sm90 or get_platform().is_sm100 else "triton" - - @_register_for( "InklingForConditionalGeneration", "InklingForConditionalGenerationMTP", @@ -75,7 +66,10 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict: # (mirrors the MiniMax-M3 SM100 fa4-default above); an explicit # --attention-backend / --prefill/decode-attention-backend still wins. if is_attention_backend_not_set(cfg): - inkling_attn_backend = resolve_inkling_attention_backend(server_args) + if get_platform().is_sm90 or get_platform().is_sm100: + inkling_attn_backend = "fa4" + else: + inkling_attn_backend = "triton" overrides["attention_backend"] = inkling_attn_backend logger.info( f"Use {inkling_attn_backend} as the attention backend for Inkling " diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index e8d60865d35e..bc2324999e3e 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -4,12 +4,8 @@ from types import SimpleNamespace from unittest.mock import patch -from sglang.srt.arg_groups.cuda_graph_hook import ( - apply_cuda_graph_compatibility, - apply_inkling_prefill_cuda_graph_default, -) +from sglang.srt.arg_groups.cuda_graph_hook import apply_cuda_graph_compatibility from sglang.srt.arg_groups.model_hook import handle_model_capability_adjustments -from sglang.srt.arg_groups.model_overrides.inkling import _inkling_overrides from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec from sglang.srt.configs.model_config import ( @@ -107,43 +103,6 @@ def test_breakable_prefill_takes_nonzero_prefix_on_cuda_only(self): with patch(target, return_value=False): self.assertFalse(runner.can_run_graph(forward_batch)) - def test_inkling_full_prefill_graph_follows_the_attention_backend(self): - # (platform, explicit backend flags) -> (attention backend, prefill graph) - cases = [ - (dict(is_sm90=True, is_sm100=False), {}, "fa4", Backend.FULL), - (dict(is_sm90=False, is_sm100=True), {}, "fa4", Backend.FULL), - (dict(is_sm90=False, is_sm100=False), {}, "triton", None), - ( - dict(is_sm90=True, is_sm100=False), - dict(attention_backend="triton"), - None, - None, - ), - ( - dict(is_sm90=True, is_sm100=False), - dict(attention_backend="fa4", prefill_attention_backend="triton"), - None, - None, - ), - ] - for platform, flags, attention_backend, prefill_backend in cases: - with self.subTest(platform=platform, flags=flags): - args = ServerArgs(model_path="dummy", **flags) - args._model_config = SimpleNamespace( - hf_config=SimpleNamespace( - architectures=["InklingForConditionalGeneration"] - ), - ) - with override_platform(is_cuda=True, **platform): - overrides = _inkling_overrides(args, None) - apply_inkling_prefill_cuda_graph_default(args) - - self.assertEqual(overrides.get("attention_backend"), attention_backend) - self.assertEqual( - resolution_result(args, "cuda_graph_backend_prefill"), - prefill_backend, - ) - def test_embedding_gemma_forces_breakable_prefill(self): args = ServerArgs(model_path="dummy") args._model_config = SimpleNamespace( From d94e9b594a1d8ecd3023b39402aadaddfd84fc8a Mon Sep 17 00:00:00 2001 From: alphabetc1 <2508695655@qq.com> Date: Sun, 11 Oct 2026 11:50:25 +0800 Subject: [PATCH 4/4] refactor: keep the Inkling prefill-graph hook unchanged --- .../sglang/srt/arg_groups/cuda_graph_hook.py | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 4fb8b8879582..2c325b6c3dcc 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -22,7 +22,6 @@ default_cuda_graph_config, with_phase, ) -from sglang.srt.runtime_context import get_platform from sglang.srt.utils.common import ( parse_connector_type, ) @@ -324,24 +323,15 @@ def apply_inkling_prefill_cuda_graph_default(server_args: Any): ): return arch = model_config_of(server_args).hf_config.architectures[0] - if arch not in ( + if arch in ( "InklingForConditionalGeneration", "InklingForConditionalGenerationMTP", ): - return - # Only fa4 captures EXTEND; an unset backend defaults to fa4 on SM90/SM100. - prefill_backend, _ = attention_backends_of(cfg) - if prefill_backend is None: - platform = get_platform() - if not (platform.is_sm90 or platform.is_sm100): - return - elif prefill_backend != "fa4": - return - declare_resolution( - server_args, - "_apply_inkling_prefill_cuda_graph_default", - cuda_graph_backend_prefill=Backend.FULL, - ) + declare_resolution( + server_args, + "_apply_inkling_prefill_cuda_graph_default", + cuda_graph_backend_prefill=Backend.FULL, + ) def apply_muse_glimmer_prefill_cuda_graph_max_bs_default(server_args: Any):