Modify export to use unified tokenizer type

Author: srikary12

torchchat/export.py

@@ -7,6 +7,7 @@
 import os
 from typing import Dict, Optional
 
+from tokenizer.tokenizer_type import TokenizerType
 import torch
 import torch._inductor
 import torch.nn as nn
@@ -482,7 +483,7 @@ def main(args):
 
     if tokenizer_args is None:
         tokenizer_type = "0"
-    elif tokenizer_args.is_sentencepiece():
+    elif tokenizer_args == TokenizerType.SENTENCEPIECE:
         tokenizer_type = "2"  # Corresponding to llama2
     else:
         tokenizer_type = "3"  # Corresponding to llama3