diff --git a/tensorrt_llm/_torch/models/modeling_kimi_k25.py b/tensorrt_llm/_torch/models/modeling_kimi_k25.py index 6c536effb8f8..314ba47fdc46 100644 --- a/tensorrt_llm/_torch/models/modeling_kimi_k25.py +++ b/tensorrt_llm/_torch/models/modeling_kimi_k25.py @@ -1052,6 +1052,13 @@ def __init__( config, "media_placeholder_token_id", _MEDIA_PLACEHOLDER_TOKEN_ID ) + # transformers 5.5.x ``AutoTokenizer`` may route K2.5 to the Rust + # fast backend, which BPE-splits ``<|media_pad|>`` / ``<|im_user|>`` + # / etc. instead of mapping them to their canonical IDs. Force the + # K2.5 slow ``TikTokenTokenizer`` for deterministic tokenization. + # See NVBug 6182617. + self._ensure_k25_slow_tokenizer() + @property def config(self) -> PretrainedConfig: return self._config @@ -1168,6 +1175,35 @@ def get_num_tokens_per_video(self, *, video: List, **kwargs) -> int: total_tokens += self.get_num_tokens_per_image(image=chunk[0]) return total_tokens + def _ensure_k25_slow_tokenizer(self) -> None: + """Override ``self._tokenizer`` and ``self._processor.tokenizer`` + with the model's slow ``TikTokenTokenizer``. + + Done unconditionally because transformers 5.5.x's ``AutoTokenizer`` + sometimes returns a Rust fast backend that BPE-splits K2.5 special + tokens instead of preserving their canonical IDs. The slow class' + ``tokens_trie`` always splits them correctly. See NVBug 6182617. + """ + from transformers.dynamic_module_utils import get_class_from_dynamic_module + + slow_cls = get_class_from_dynamic_module( + "tokenization_kimi.TikTokenTokenizer", + self._model_path, + ) + slow_tok = slow_cls.from_pretrained(self._model_path, trust_remote_code=True) + + logger.info( + "K2.5 InputProcessor forcing slow TikTokenTokenizer " + "(originally %s). See NVBug 6182617.", + type(self._tokenizer).__name__, + ) + + self._tokenizer = slow_tok + # Image-only path uses ``self._processor.tokenizer`` (an + # independent instance from ``AutoProcessor``); swap it too. + if getattr(self._processor, "tokenizer", None) is not None: + self._processor.tokenizer = slow_tok + @torch.inference_mode() def __call__( self, diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 39ce3d05d54d..4638f01ecdda 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -524,9 +524,13 @@ class TestKimiK25(LlmapiAccuracyTestHarness): ) def test_nvfp4(self, ep_size, attention_dp): """NVFP4 accuracy on MMMU benchmark (8x B200).""" + # Do not pass ``max_num_tokens`` here: keep it at the LLM default + # (8192) so the resolved ``moe_max_num_tokens`` stays at + # ``8192 * dp_size`` and the per-call fused_moe workspace fits + # within activation headroom. Raising it pushed the workspace + # past the fragmented allocator budget on dep8 (NVBug 6182617). with LLM( self.MODEL_PATH, - max_num_tokens=self.MAX_NUM_TOKENS, kv_cache_config=self.kv_cache_config, tensor_parallel_size=8, pipeline_parallel_size=1, diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 64452650c4dd..8c9d1084dc88 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -154,7 +154,7 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp3_no_adp] TIMEOUT (60) - accuracy/test_disaggregated_serving.py::TestDeepSeekV32Exp::test_auto_dtype[False] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] TIMEOUT (60) - - accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (60) + - accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (120) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-trtllm] TIMEOUT (60) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 3de7a088c8ec..bf46530fabb1 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -144,7 +144,6 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] SKIP (h accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[throughput_latency] SKIP (https://nvbugs/6177390) accuracy/test_llm_api_pytorch.py::TestQwen3_8B::test_fp8_block_scales_early_first_token_response SKIP (https://nvbugs/6200128) accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized SKIP (https://nvbugs/6189416) -accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] SKIP (https://nvbugs/6182617) accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] SKIP (https://nvbugs/6181383) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6143787) accuracy/test_llm_api_pytorch_multimodal.py::TestQwen3VL_MOE::test_auto_dtype SKIP (https://nvbugs/6114464)