From e8c6a40666cdfbb775f29f8db4d4a8943b3ea1ff Mon Sep 17 00:00:00 2001 From: Umar Butler Date: Wed, 18 Sep 2024 18:25:02 +1000 Subject: [PATCH 1/2] Fix ByteLevel alphabet missing when Sequence pretokenizer is used --- src/transformers/tokenization_utils_fast.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py index 724484b3b30b..5e528f978b88 100644 --- a/src/transformers/tokenization_utils_fast.py +++ b/src/transformers/tokenization_utils_fast.py @@ -824,7 +824,12 @@ def train_new_from_iterator( kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"] if tokenizer_json["model"]["type"] == "Unigram" and unk_token is not None: kwargs["unk_token"] = unk_token - if tokenizer_json["pre_tokenizer"] is not None and tokenizer_json["pre_tokenizer"]["type"] == "ByteLevel": + if tokenizer_json["pre_tokenizer"] is not None \ + and tokenizer_json["pre_tokenizer"]["type"] == "ByteLevel" \ + or tokenizer_json["pre_tokenizer"]["type"] == "Sequence" and \ + "pretokenizers" in tokenizer_json["pre_tokenizer"] and \ + any(pretokenizer["type"] == "ByteLevel" + for pretokenizer in tokenizer_json["pre_tokenizer"]["pretokenizers"]): kwargs["initial_alphabet"] = pre_tokenizers_fast.ByteLevel.alphabet() trainer_class = MODEL_TO_TRAINER_MAPPING[tokenizer_json["model"]["type"]] From f631ca37ed8683e32178c040e10604ab4e40eaa1 Mon Sep 17 00:00:00 2001 From: Umar Butler Date: Sun, 22 Sep 2024 21:57:25 +1000 Subject: [PATCH 2/2] Fixed formatting with `ruff`. --- src/transformers/tokenization_utils_fast.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py index 5e528f978b88..94815caf352c 100644 --- a/src/transformers/tokenization_utils_fast.py +++ b/src/transformers/tokenization_utils_fast.py @@ -824,12 +824,16 @@ def train_new_from_iterator( kwargs["end_of_word_suffix"] = tokenizer_json["model"]["end_of_word_suffix"] if tokenizer_json["model"]["type"] == "Unigram" and unk_token is not None: kwargs["unk_token"] = unk_token - if tokenizer_json["pre_tokenizer"] is not None \ - and tokenizer_json["pre_tokenizer"]["type"] == "ByteLevel" \ - or tokenizer_json["pre_tokenizer"]["type"] == "Sequence" and \ - "pretokenizers" in tokenizer_json["pre_tokenizer"] and \ - any(pretokenizer["type"] == "ByteLevel" - for pretokenizer in tokenizer_json["pre_tokenizer"]["pretokenizers"]): + if ( + tokenizer_json["pre_tokenizer"] is not None + and tokenizer_json["pre_tokenizer"]["type"] == "ByteLevel" + or tokenizer_json["pre_tokenizer"]["type"] == "Sequence" + and "pretokenizers" in tokenizer_json["pre_tokenizer"] + and any( + pretokenizer["type"] == "ByteLevel" + for pretokenizer in tokenizer_json["pre_tokenizer"]["pretokenizers"] + ) + ): kwargs["initial_alphabet"] = pre_tokenizers_fast.ByteLevel.alphabet() trainer_class = MODEL_TO_TRAINER_MAPPING[tokenizer_json["model"]["type"]]