From abb019bc6cc82a2ac55a8cc66d6329a9bdc33925 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 10 Apr 2026 12:41:40 +0200 Subject: [PATCH] Fix Kimi-K2.5 tokenizer regression and _patch_mistral_regex AttributeError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #45356 Remove `kimi_k25` from `MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS` — its remote `TikTokenTokenizer` is the only correct backend (no `tokenizer.json`, non-sequential added-token IDs that `TokenizersBackend` cannot reproduce). Also fix `_patch_mistral_regex`: the method receives the raw `tokenizers.Tokenizer` object, which has `.pre_tokenizer` directly, not `.backend_tokenizer.pre_tokenizer`. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/transformers/models/auto/tokenization_auto.py | 1 - src/transformers/tokenization_utils_tokenizers.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 1b38f2e7a3f1..3d12426caa6f 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -358,7 +358,6 @@ "internvl_chat", "jamba", "janus", - "kimi_k25", "llava", "llava_next", "minicpmv", diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index b516a777ecf1..fcd82078295e 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -1360,11 +1360,11 @@ def is_base_mistral(model_id: str) -> bool: ), behavior="isolated", ) - current_pretokenizer = tokenizer.backend_tokenizer.pre_tokenizer + current_pretokenizer = tokenizer.pre_tokenizer # Check if it's already a Sequence if isinstance(current_pretokenizer, tokenizers.pre_tokenizers.Sequence): # Replace the first element (the Split pattern) - tokenizer.backend_tokenizer.pre_tokenizer[0] = split_pretokenizer + tokenizer.pre_tokenizer[0] = split_pretokenizer else: # Replace Metaspace with ByteLevel when adding Split, as Metaspace(split=False) doesn't # work correctly with the Split pre-tokenizer and causes spaces to be lost during encoding @@ -1374,7 +1374,7 @@ def is_base_mistral(model_id: str) -> bool: ) # Not a Sequence, so create one with Split + current pretokenizer - tokenizer.backend_tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence( + tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence( [ split_pretokenizer, current_pretokenizer,