ITN runtime. (#2001)

* wenet_api and decoder_main support ITN.

* remove resource dir

* turn websocket cmake on

* fix cpplint

* fix clang-format

* fix clang-format

* fix clang-format

* fix CRLF.

* try to add wetext as subproject, but failed. Because wetext and wenet both have utils and utils/string.h

* add wetext subproject, intergrate ITN.

* add fst file exist check.

* include file.h in params.h

---------

Co-authored-by: dujing <[email protected]>

Author: duj12

runtime/core/api/wenet_api.cc

@@ -65,8 +65,8 @@ class Recognizer {
 
     std::string fst_path = wenet::JoinPath(model_dir, "TLG.fst");
     if (wenet::FileExists(fst_path)) {  // With LM
-      resource_->fst = std::shared_ptr<fst::Fst<fst::StdArc>>(
-          fst::Fst<fst::StdArc>::Read(fst_path));
+      resource_->fst = std::shared_ptr<fst::VectorFst<fst::StdArc>>(
+          fst::VectorFst<fst::StdArc>::Read(fst_path));
 
       std::string symbol_path = wenet::JoinPath(model_dir, "words.txt");
       CHECK(wenet::FileExists(symbol_path));
@@ -79,7 +79,30 @@ class Recognizer {
     // Context config init
     context_config_ = std::make_shared<wenet::ContextConfig>();
     decode_options_ = std::make_shared<wenet::DecodeOptions>();
+
+    // PostProcessor
     post_process_opts_ = std::make_shared<wenet::PostProcessOptions>();
+    if (language_ == "chs") {  // TODO(Binbin Zhang): CJK(chs, jp, kr)
+      post_process_opts_->language_type = wenet::kMandarinEnglish;
+    } else {
+      post_process_opts_->language_type = wenet::kIndoEuropean;
+    }
+    resource_->post_processor =
+        std::make_shared<wenet::PostProcessor>(*post_process_opts_);
+    // Optional: ITN
+    std::string itn_tagger_path =
+        wenet::JoinPath(model_dir, "zh_itn_tagger.fst");
+    std::string itn_verbalizer_path =
+        wenet::JoinPath(model_dir, "zh_itn_verbalizer.fst");
+    if (wenet::FileExists(itn_tagger_path) &&
+        wenet::FileExists(itn_verbalizer_path)) {
+      LOG(INFO) << "Reading ITN fst";
+      post_process_opts_->itn = true;
+      auto postprocessor =
+          std::make_shared<wenet::PostProcessor>(*post_process_opts_);
+      postprocessor->InitITNResource(itn_tagger_path, itn_verbalizer_path);
+      resource_->post_processor = postprocessor;
+    }
   }
 
   void Reset() {
@@ -101,14 +124,7 @@ class Recognizer {
       context_graph->BuildContextGraph(context_, resource_->symbol_table);
       resource_->context_graph = context_graph;
     }
-    // PostProcessor
-    if (language_ == "chs") {  // TODO(Binbin Zhang): CJK(chs, jp, kr)
-      post_process_opts_->language_type = wenet::kMandarinEnglish;
-    } else {
-      post_process_opts_->language_type = wenet::kIndoEuropean;
-    }
-    resource_->post_processor =
-        std::make_shared<wenet::PostProcessor>(*post_process_opts_);
+
     // Init decode options
     decode_options_->chunk_size = chunk_size_;
     // Init decoder

runtime/core/bin/CMakeLists.txt

@@ -7,10 +7,10 @@ endif()
 add_executable(label_checker_main label_checker_main.cc)
 target_link_libraries(label_checker_main PUBLIC decoder)
 
-# if(TORCH)
-#  add_executable(api_main api_main.cc)
-#  target_link_libraries(api_main PUBLIC wenet_api)
-# endif()
+if(TORCH)
+ add_executable(api_main api_main.cc)
+ target_link_libraries(api_main PUBLIC wenet_api)
+endif()
 
 if(WEBSOCKET)
   add_executable(websocket_client_main websocket_client_main.cc)

runtime/core/cmake/openfst.cmake

@@ -29,8 +29,10 @@ if(NOT ANDROID)
   # To build openfst with gflags and glog, we comment out some vars of {flags, log}.h and flags.cc.
   set(openfst_SOURCE_DIR ${fc_base}/openfst-src CACHE PATH "OpenFST source directory")
     FetchContent_Declare(openfst
-      URL           https://github.com/kkm000/openfst/archive/refs/tags/win/1.6.5.1.tar.gz
-      URL_HASH      SHA256=02c49b559c3976a536876063369efc0e41ab374be1035918036474343877046e
+      URL           https://github.com/kkm000/openfst/archive/refs/tags/win/1.7.2.1.tar.gz
+      URL_HASH      SHA256=e04e1dabcecf3a687ace699ccb43a8a27da385777a56e69da6e103344cc66bca
+      #URL           https://github.com/kkm000/openfst/archive/refs/tags/win/1.6.5.1.tar.gz
+      #URL_HASH      SHA256=02c49b559c3976a536876063369efc0e41ab374be1035918036474343877046e
       PATCH_COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR}
     )
     FetchContent_MakeAvailable(openfst)

runtime/core/cmake/wetextprocessing.cmake

@@ -0,0 +1,10 @@
+FetchContent_Declare(wetextprocessing
+  GIT_REPOSITORY https://github.com/wenet-e2e/WeTextProcessing.git
+  GIT_TAG        origin/master
+)
+FetchContent_MakeAvailable(wetextprocessing)
+include_directories(${wetextprocessing_SOURCE_DIR}/runtime )
+add_subdirectory(${wetextprocessing_SOURCE_DIR}/runtime/utils)
+add_subdirectory(${wetextprocessing_SOURCE_DIR}/runtime/processor)
+
+

runtime/core/decoder/asr_decoder.h

@@ -92,7 +92,7 @@ enum DecodeState {
 struct DecodeResource {
   std::shared_ptr<AsrModel> model = nullptr;
   std::shared_ptr<fst::SymbolTable> symbol_table = nullptr;
-  std::shared_ptr<fst::Fst<fst::StdArc>> fst = nullptr;
+  std::shared_ptr<fst::VectorFst<fst::StdArc>> fst = nullptr;
   std::shared_ptr<fst::SymbolTable> unit_table = nullptr;
   std::shared_ptr<ContextGraph> context_graph = nullptr;
   std::shared_ptr<PostProcessor> post_processor = nullptr;
@@ -140,7 +140,7 @@ class AsrDecoder {
   std::shared_ptr<PostProcessor> post_processor_;
   std::shared_ptr<ContextGraph> context_graph_;
 
-  std::shared_ptr<fst::Fst<fst::StdArc>> fst_ = nullptr;
+  std::shared_ptr<fst::VectorFst<fst::StdArc>> fst_ = nullptr;
   // output symbol table
   std::shared_ptr<fst::SymbolTable> symbol_table_;
   // e2e unit symbol table

runtime/core/decoder/params.h

@@ -39,6 +39,7 @@
 #endif
 #include "frontend/feature_pipeline.h"
 #include "post_processor/post_processor.h"
+#include "utils/file.h"
 #include "utils/flags.h"
 #include "utils/string.h"
 
@@ -65,6 +66,11 @@ DEFINE_int32(sample_rate, 16000, "sample rate for audio");
 // TLG fst
 DEFINE_string(fst_path, "", "TLG fst path");
 
+// ITN fst
+DEFINE_string(itn_model_dir, "",
+              "fst based ITN model dir, "
+              "should contain itn_tagger.fst and itn_verbalizer.fst");
+
 // DecodeOptions flags
 DEFINE_int32(chunk_size, 16, "decoding chunk size");
 DEFINE_int32(num_left_chunks, -1, "left chunks in decoding");
@@ -203,8 +209,8 @@ std::shared_ptr<DecodeResource> InitDecodeResourceFromFlags() {
   if (!FLAGS_fst_path.empty()) {  // With LM
     CHECK(!FLAGS_dict_path.empty());
     LOG(INFO) << "Reading fst " << FLAGS_fst_path;
-    auto fst = std::shared_ptr<fst::Fst<fst::StdArc>>(
-        fst::Fst<fst::StdArc>::Read(FLAGS_fst_path));
+    auto fst = std::shared_ptr<fst::VectorFst<fst::StdArc>>(
+        fst::VectorFst<fst::StdArc>::Read(FLAGS_fst_path));
     CHECK(fst != nullptr);
     resource->fst = fst;
 
@@ -237,6 +243,23 @@ std::shared_ptr<DecodeResource> InitDecodeResourceFromFlags() {
   post_process_opts.lowercase = FLAGS_lowercase;
   resource->post_processor =
       std::make_shared<PostProcessor>(std::move(post_process_opts));
+
+  if (!FLAGS_itn_model_dir.empty()) {  // With ITN
+    std::string itn_tagger_path =
+        wenet::JoinPath(FLAGS_itn_model_dir, "zh_itn_tagger.fst");
+    std::string itn_verbalizer_path =
+        wenet::JoinPath(FLAGS_itn_model_dir, "zh_itn_verbalizer.fst");
+    if (wenet::FileExists(itn_tagger_path) &&
+        wenet::FileExists(itn_verbalizer_path)) {
+      LOG(INFO) << "Reading ITN fst" << FLAGS_itn_model_dir;
+      post_process_opts.itn = true;
+      auto postprocessor =
+          std::make_shared<wenet::PostProcessor>(std::move(post_process_opts));
+      postprocessor->InitITNResource(itn_tagger_path, itn_verbalizer_path);
+      resource->post_processor = postprocessor;
+    }
+  }
+
   return resource;
 }
 

runtime/core/patch/openfst/src/test/CMakeLists.txt

@@ -0,0 +1,55 @@
+add_executable(fst_test
+        fst_test.cc
+        ../include/fst/test/fst_test.h
+        )
+target_link_libraries(fst_test fst ${CMAKE_DL_LIBS})
+set_target_properties(fst_test PROPERTIES FOLDER test)
+add_test(NAME fst_test-test COMMAND fst_test)
+
+add_executable(weight_test
+        weight_test.cc
+        ../include/fst/test/weight-tester.h
+        )
+target_link_libraries(weight_test fst ${CMAKE_DL_LIBS})
+set_target_properties(weight_test PROPERTIES FOLDER test)
+add_test(NAME weight_test-test COMMAND weight_test)
+
+add_executable(algo_test_log algo_test.cc ../include/fst/test/algo_test.h ../include/fst/test/rand-fst.h)
+target_link_libraries(algo_test_log fst ${CMAKE_DL_LIBS})
+target_compile_definitions(algo_test_log
+        PRIVATE TEST_LOG=1)
+set_target_properties(algo_test_log PROPERTIES FOLDER test)
+add_test(NAME algo_test_log-test COMMAND algo_test_log)
+
+
+add_executable(algo_test_tropical algo_test.cc ../include/fst/test/algo_test.h ../include/fst/test/rand-fst.h)
+target_link_libraries(algo_test_tropical fst ${CMAKE_DL_LIBS})
+target_compile_definitions(algo_test_tropical
+        PRIVATE TEST_TROPICAL=1)
+set_target_properties(algo_test_tropical PROPERTIES FOLDER test)
+add_test(NAME algo_test_tropical-test COMMAND algo_test_tropical)
+
+
+add_executable(algo_test_minmax algo_test.cc ../include/fst/test/algo_test.h ../include/fst/test/rand-fst.h)
+target_link_libraries(algo_test_minmax fst ${CMAKE_DL_LIBS})
+target_compile_definitions(algo_test_minmax
+        PRIVATE TEST_MINMAX=1)
+set_target_properties(algo_test_minmax PROPERTIES FOLDER test)
+add_test(NAME algo_test_minmax-test COMMAND algo_test_minmax)
+
+
+add_executable(algo_test_lexicographic algo_test.cc ../include/fst/test/algo_test.h ../include/fst/test/rand-fst.h)
+target_link_libraries(algo_test_lexicographic fst ${CMAKE_DL_LIBS})
+target_compile_definitions(algo_test_lexicographic
+        PRIVATE TEST_LEXICOGRAPHIC=1)
+set_target_properties(algo_test_lexicographic PROPERTIES FOLDER test)
+add_test(NAME algo_test_lexicographic-test COMMAND algo_test_lexicographic)
+
+
+add_executable(algo_test_power algo_test.cc ../include/fst/test/algo_test.h ../include/fst/test/rand-fst.h)
+target_link_libraries(algo_test_power fst ${CMAKE_DL_LIBS})
+target_compile_definitions(algo_test_power
+        PRIVATE TEST_POWER=1)
+set_target_properties(algo_test_power PROPERTIES FOLDER test)
+add_test(NAME algo_test_power-test COMMAND algo_test_power)
+

runtime/core/post_processor/CMakeLists.txt

@@ -1,4 +1,8 @@
 add_library(post_processor STATIC
   post_processor.cc
 )
-target_link_libraries(post_processor PUBLIC utils)
+if(ITN)
+  target_link_libraries(post_processor PUBLIC utils wetext_processor)
+else()
+  target_link_libraries(post_processor PUBLIC utils)
+endif()

runtime/core/post_processor/post_processor.cc

@@ -1,4 +1,5 @@
 // Copyright (c) 2021 Xingchen Song [email protected]
+//               2023 Jing Du ([email protected])
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,13 +14,18 @@
 // limitations under the License
 
 #include "post_processor/post_processor.h"
-
 #include <sstream>
 #include <vector>
-
+#include "processor/wetext_processor.h"
 #include "utils/string.h"
 
 namespace wenet {
+void PostProcessor::InitITNResource(const std::string& tagger_path,
+                                    const std::string& verbalizer_path) {
+  auto itn_processor =
+      std::make_shared<wetext::Processor>(tagger_path, verbalizer_path);
+  itn_resource = itn_processor;
+}
 
 std::string PostProcessor::ProcessSpace(const std::string& str) {
   std::string result = str;
@@ -56,10 +62,34 @@ std::string PostProcessor::ProcessSpace(const std::string& str) {
   return result;
 }
 
+std::string del_substr(const std::string& str, const std::string& sub) {
+  std::string result = str;
+  int pos = 0;
+  while (string::npos != (pos = result.find(sub))) {
+    result.erase(pos, sub.size());
+  }
+  return result;
+}
+
+std::string PostProcessor::ProcessSymbols(const std::string& str) {
+  std::string result = str;
+  result = del_substr(result, "<unk>");
+  result = del_substr(result, "<context>");
+  result = del_substr(result, "</context>");
+  return result;
+}
+
 std::string PostProcessor::Process(const std::string& str, bool finish) {
   std::string result;
-  result = ProcessSpace(str);
-  // TODO(xcsong): do itn/punctuation if finish == true
+  // remove symbols with "<>" first
+  result = ProcessSymbols(str);
+  result = ProcessSpace(result);
+  // TODO(xcsong): do punctuation if finish == true
+  if (finish == true && opts_.itn) {
+    if (nullptr != itn_resource) {
+      result = itn_resource->Normalize(result);
+    }
+  }
   return result;
 }
 

runtime/core/post_processor/post_processor.h

@@ -1,4 +1,5 @@
 // Copyright (c) 2021 Xingchen Song [email protected]
+//               2023 Jing Du ([email protected])
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,7 +19,7 @@
 #include <memory>
 #include <string>
 #include <utility>
-
+#include "processor/wetext_processor.h"
 #include "utils/utils.h"
 
 namespace wenet {
@@ -43,10 +44,10 @@ struct PostProcessOptions {
   LanguageType language_type = kMandarinEnglish;
   // whether lowercase letters are required
   bool lowercase = true;
+  bool itn = false;
 };
 
-// TODO(xcsong): add itn/punctuation related resource
-struct PostProcessResource {};
+// TODO(xcsong): add punctuation related resource
 
 // Post Processor
 class PostProcessor {
@@ -57,11 +58,15 @@ class PostProcessor {
   std::string Process(const std::string& str, bool finish);
   // process spaces according to configurations
   std::string ProcessSpace(const std::string& str);
-  // TODO(xcsong): add itn/punctuation
-  // void InverseTN(const std::string& str);
+  std::string ProcessSymbols(const std::string& str);
+  // TODO(xcsong): add punctuation
   // void Punctuate(const std::string& str);
 
+  void InitITNResource(const std::string& tagger_path,
+                       const std::string& verbalizer_path);
+
  private:
+  std::shared_ptr<wetext::Processor> itn_resource = nullptr;
   const PostProcessOptions opts_;
 
  public:

runtime/libtorch/CMakeLists.txt

@@ -4,7 +4,7 @@ project(wenet VERSION 0.1)
 
 option(CXX11_ABI "whether to use CXX11_ABI libtorch" OFF)
 option(GRAPH_TOOLS "whether to build TLG graph tools" OFF)
-option(BUILD_TESTING "whether to build unit test" ON)
+option(BUILD_TESTING "whether to build unit test" OFF)
 
 option(GRPC "whether to build with gRPC" OFF)
 # TODO(Binbin Zhang): Change websocket to OFF since it depends on boost
@@ -14,7 +14,9 @@ option(HTTP "whether to build with http" OFF)
 option(TORCH "whether to build with Torch" ON)
 option(ONNX "whether to build with ONNX" OFF)
 option(GPU "whether to build with GPU" OFF)
+option(ITN "whether to use WeTextProcessing" ON)
 
+set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -g")
 set(CMAKE_VERBOSE_MAKEFILE OFF)
 
 include(FetchContent)
@@ -46,6 +48,10 @@ include_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}/kaldi
 )
 
+if(ITN)
+  include(wetextprocessing)
+endif()
+
 # Build all libraries
 add_subdirectory(utils)
 add_subdirectory(frontend)