Move conditional tiktoken import to the __init__ func (#842)

* Move conditional tiktoken import to the __init__ func * Update NeoXArgs docs automatically --------- Co-authored-by: github-actions <[email protected]>
EleutherAI · Mar 18, 2023 · e2d8a24 · e2d8a24
1 parent 91b72d9
commit e2d8a24
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 7 deletions.
diff --git a/configs/neox_arguments.md b/configs/neox_arguments.md
@@ -111,7 +111,7 @@ Logging Arguments
 
 - **git_hash**: str
 
-    Default = 7d682df
+    Default = 6dd0344
 
     current git hash of repository
 

diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -353,13 +353,13 @@ def eod(self):
 class TiktokenTokenizer(AbstractTokenizer):
     """Tokenizer from OpenAI's tiktoken implementation"""
 
-    try:
-        import tiktoken
-    except ModuleNotFoundError:
-        print("Please install tiktoken: (https://github.com/openai/tiktoken)")
-        raise Exception
-
     def __init__(self, vocab_file):
+        try:
+            import tiktoken
+        except ModuleNotFoundError:
+            print("Please install tiktoken: (https://github.com/openai/tiktoken)")
+            raise Exception
+
         name = "TiktokenTokenizer"
         super().__init__(name)