diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 1b38f2e7a3f1..3d12426caa6f 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -358,7 +358,6 @@ "internvl_chat", "jamba", "janus", - "kimi_k25", "llava", "llava_next", "minicpmv", diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index b516a777ecf1..fcd82078295e 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -1360,11 +1360,11 @@ def is_base_mistral(model_id: str) -> bool: ), behavior="isolated", ) - current_pretokenizer = tokenizer.backend_tokenizer.pre_tokenizer + current_pretokenizer = tokenizer.pre_tokenizer # Check if it's already a Sequence if isinstance(current_pretokenizer, tokenizers.pre_tokenizers.Sequence): # Replace the first element (the Split pattern) - tokenizer.backend_tokenizer.pre_tokenizer[0] = split_pretokenizer + tokenizer.pre_tokenizer[0] = split_pretokenizer else: # Replace Metaspace with ByteLevel when adding Split, as Metaspace(split=False) doesn't # work correctly with the Split pre-tokenizer and causes spaces to be lost during encoding @@ -1374,7 +1374,7 @@ def is_base_mistral(model_id: str) -> bool: ) # Not a Sequence, so create one with Split + current pretokenizer - tokenizer.backend_tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence( + tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Sequence( [ split_pretokenizer, current_pretokenizer,