[text] rm WenetTokenizer (#2218)

* [text] rm WenetTokenizer * [text] fix ut
wenet-e2e · Dec 12, 2023 · 92d9b66 · 92d9b66
1 parent d2b337d
commit 92d9b66
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 272 deletions.
diff --git a/test/wenet/dataset/test_processor.py b/test/wenet/dataset/test_processor.py
@@ -1,7 +1,7 @@
 import pytest
 
 import wenet.dataset.processor as processor
-from wenet.text.wenet_tokenizer import WenetTokenizer
+from wenet.utils.init_tokenizer import init_tokenizer
 
 
 @pytest.mark.parametrize("symbol_table_path", [
@@ -139,9 +139,8 @@ def test_tokenize(symbol_table_path):
             "label": [24, 46, 2, 43, 1, 35, 27, 7, 56]
         }]
 
-    tokenizer = WenetTokenizer(symbol_table_path,
-                               bpe_model,
-                               split_with_space=False)
+    configs = {'split_with_space': False}
+    tokenizer = init_tokenizer(configs, symbol_table_path, bpe_model)
     outs = processor.tokenize(txts, tokenizer)
     for (hyp, ref) in zip(outs, refs):
         assert (len(hyp["tokens"]) == len(ref["tokens"]))

diff --git a/test/wenet/text/test_wenet_tokenzier.py b/test/wenet/text/test_wenet_tokenzier.py
diff --git a/wenet/text/wenet_tokenizer.py b/wenet/text/wenet_tokenizer.py
diff --git a/wenet/utils/init_tokenizer.py b/wenet/utils/init_tokenizer.py
@@ -9,8 +9,7 @@ def init_tokenizer(configs,
                    bpe_model=None,
                    non_lang_syms=None) -> BaseTokenizer:
     # TODO:
-    # 1 huggface tokenizer
-    # 2 paraformer tokenizer
+    # 1 paraformer tokenizer
 
     if configs.get("whisper", False):
         tokenizer = WhisperTokenizer(