diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot.py b/src/transformers/models/blenderbot/tokenization_blenderbot.py index 157d495a7af9..ace4afc6d503 100644 --- a/src/transformers/models/blenderbot/tokenization_blenderbot.py +++ b/src/transformers/models/blenderbot/tokenization_blenderbot.py @@ -99,8 +99,9 @@ class BlenderbotTokenizer(PreTrainedTokenizer): ``` >>> from transformers import BlenderbotTokenizer >>> tokenizer = BlenderbotTokenizer.from_pretrained("facebook/blenderbot-3B") + >>> tokenizer.add_prefix_space = False >>> tokenizer("Hello world")['input_ids'] - [6950, 1085, 2] + [47, 921, 86, 1085, 2] >>> tokenizer(" Hello world")['input_ids'] [6950, 1085, 2] ```