diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md index 6bb2ded07..416ad5f83 100644 --- a/configs/neox_arguments.md +++ b/configs/neox_arguments.md @@ -111,7 +111,7 @@ Logging Arguments - **git_hash**: str - Default = 7d682df + Default = 6dd0344 current git hash of repository diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py index a9b74fc23..7c81d0ff4 100644 --- a/megatron/tokenizer/tokenizer.py +++ b/megatron/tokenizer/tokenizer.py @@ -353,13 +353,13 @@ def eod(self): class TiktokenTokenizer(AbstractTokenizer): """Tokenizer from OpenAI's tiktoken implementation""" - try: - import tiktoken - except ModuleNotFoundError: - print("Please install tiktoken: (https://github.com/openai/tiktoken)") - raise Exception - def __init__(self, vocab_file): + try: + import tiktoken + except ModuleNotFoundError: + print("Please install tiktoken: (https://github.com/openai/tiktoken)") + raise Exception + name = "TiktokenTokenizer" super().__init__(name)