diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py index d08cb9ed5a42..f681b5496527 100644 --- a/tensorrt_llm/tokenizer/tokenizer.py +++ b/tensorrt_llm/tokenizer/tokenizer.py @@ -10,6 +10,19 @@ from .._utils import nvtx_range_debug from ..logger import logger +# Transformers 5.x moved ``bytes_to_unicode`` out of +# ``transformers.models.gpt2.tokenization_gpt2`` into +# ``transformers.convert_slow_tokenizer``. Some ``trust_remote_code=True`` +# checkpoints (e.g. Kimi-K2's ``tokenization_kimi.py``) still import it from +# the legacy location; re-export the symbol so those tokenizers keep loading. +try: + from transformers.models.gpt2 import tokenization_gpt2 as _gpt2_mod + if not hasattr(_gpt2_mod, "bytes_to_unicode"): + from transformers.convert_slow_tokenizer import bytes_to_unicode + _gpt2_mod.bytes_to_unicode = bytes_to_unicode +except ImportError: + pass + # Aliases for built-in custom tokenizers. TOKENIZER_ALIASES = { "deepseek_v32": "tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer", diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 923063a16176..26ea91b9d458 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -95,8 +95,6 @@ accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-ep4-cutl accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_chunked_prefill[cutlass-auto] SKIP (https://nvbugs/5596343) accuracy/test_llm_api_pytorch.py::TestKanana_Instruct::test_auto_dtype SKIP (https://nvbugs/6209806) -accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_async_cancel SKIP (https://nvbugs/6160085) -accuracy/test_llm_api_pytorch.py::TestKimiK2::test_nvfp4_longseq_trtllm_moe_stress SKIP (https://nvbugs/6160085) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=False-enable_padding=False-disable_overlap_scheduler=True-sampler_async_worker=False] SKIP (https://nvbugs/6141653) accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_auto_dtype_beam_search[enable_cuda_graph=True-enable_padding=False-disable_overlap_scheduler=False-sampler_async_worker=False] SKIP (https://nvbugs/6141653)