refactor: clean up GPT2Tokenizer implementation by removing unused co…

…de and comments

Signed-off-by: -LAN- <[email protected]>

api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py

@@ -1,4 +1,3 @@
-from os.path import abspath, dirname, join
 from threading import Lock
 from typing import Any
 
@@ -21,7 +20,7 @@ def _get_num_tokens_by_gpt2(text: str) -> int:
     @staticmethod
     def get_num_tokens(text: str) -> int:
         # Because this process needs more cpu resource, we turn this back before we find a better way to handle it.
-        # 
+        #
         # future = _executor.submit(GPT2Tokenizer._get_num_tokens_by_gpt2, text)
         # result = future.result()
         # return cast(int, result)
@@ -32,10 +31,11 @@ def get_encoder() -> Any:
         global _tokenizer, _lock
         with _lock:
             if _tokenizer is None:
-                base_path = abspath(__file__)
-                gpt2_tokenizer_path = join(dirname(base_path), "gpt2")
-                # _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path)
                 # Try to use tiktoken to get the tokenizer because it is faster
+                #
                 _tokenizer = tiktoken.get_encoding("gpt2")
+                # base_path = abspath(__file__)
+                # gpt2_tokenizer_path = join(dirname(base_path), "gpt2")
+                # _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path)
 
             return _tokenizer