diff --git a/python/sglang/srt/arg_groups/memory_hook.py b/python/sglang/srt/arg_groups/memory_hook.py index 73f32960d043..87e4634c16ee 100644 --- a/python/sglang/srt/arg_groups/memory_hook.py +++ b/python/sglang/srt/arg_groups/memory_hook.py @@ -231,13 +231,11 @@ def handle_gpu_memory_settings(server_args: Any): decode_cuda_graph_config.max_bs = cfg.torch_compile_max_bs if prefill_cuda_graph_config.max_bs is None: - # Refer to pr #15927, by default we set the prefill max_bs to the chunked prefill size. - # For MLA backend, the introduction of piecewise cuda graph will influence the kernel dispatch difference compared to the original mode. - # To avoid the performance regression, we set max_bs to 2048 by default. + # Default to the chunked prefill size; cap MLA capture at 8192 tokens. if not use_mla_backend(server_args): prefill_cuda_graph_config.max_bs = cfg.chunked_prefill_size else: - prefill_cuda_graph_config.max_bs = 2048 + prefill_cuda_graph_config.max_bs = min(8192, cfg.chunked_prefill_size) # For opt-in PP breakable graphs, capture small aggregate-token # buckets by default and leave larger forwards on the eager path.