From c2a0ed6170a9d2b50e0347a0547a3ffbefa6201e Mon Sep 17 00:00:00 2001 From: Shicheng Li Date: Wed, 13 May 2026 23:59:51 -0700 Subject: [PATCH] [None][feat] Enable 2 DSv4 perf optimizations by default Switch defaults so DeepSeek-V4 inference paths run with two previously opt-in optimizations enabled out of the box. Both remain user-disableable via the same env vars. 1. PR #13628 fused FP8 1x128 quantize + UE8M0 pack on SM100 - tensorrt_llm/_torch/custom_ops/torch_custom_ops.py - Env: TRTLLM_FUSED_FP8_QUANT_PACK (default '0' -> '1') - Disable: TRTLLM_FUSED_FP8_QUANT_PACK=0 2. PR #13629 MLA dependency-aware overlap on DSv4 - tensorrt_llm/_torch/modules/attention.py - Env: TRTLLM_MLA_EXTRA_OVERLAP (default '0' -> '1') - Disable: TRTLLM_MLA_EXTRA_OVERLAP=0 The third originally-proposed flip (use_cute_dsl_blockscaling_bmm) is dropped from this PR. The cute_dsl FP8 BMM path is also invoked for DSv3 K/V absorption BMMs on Blackwell + FP8 block-scales (the fp8_block_scaling_bmm_out dispatcher at attention.py:1161 is not gated on is_deepseek_v4), so flipping the default would change DSv3 perf behavior silently. Defer that flip until a DSv3 Blackwell-FP8 smoke confirms no regression. Signed-off-by: Shicheng Li --- tensorrt_llm/_torch/custom_ops/torch_custom_ops.py | 2 +- tensorrt_llm/_torch/modules/attention.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py index 3b3d8a0d4c6b..ff99ab961099 100644 --- a/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py +++ b/tensorrt_llm/_torch/custom_ops/torch_custom_ops.py @@ -1530,7 +1530,7 @@ def _( # deep_gemm_gen_tuning_buckets is imported from ..utils _USE_FUSED_FP8_QUANT_PACK = os.environ.get("TRTLLM_FUSED_FP8_QUANT_PACK", - "0") == "1" + "1") == "1" def _fp8_quantize_1x128_ue8m0(input: torch.Tensor, tactic: int): diff --git a/tensorrt_llm/_torch/modules/attention.py b/tensorrt_llm/_torch/modules/attention.py index f25ea20f7c86..681155029064 100644 --- a/tensorrt_llm/_torch/modules/attention.py +++ b/tensorrt_llm/_torch/modules/attention.py @@ -2195,7 +2195,7 @@ def forward_impl_with_deepseek_v4(self, # with q_b_proj + q_b_layernorm. The indexer is launched on a # dedicated stream and still uses a different aux stream for its # internal q-proj/weights-proj split. - _v4_extra_overlap = (os.environ.get("TRTLLM_MLA_EXTRA_OVERLAP", "0") + _v4_extra_overlap = (os.environ.get("TRTLLM_MLA_EXTRA_OVERLAP", "1") == "1" and self.compressor is not None and self.aux_stream is not None)