Refactor TTS Android code to support jieba for Chinese TTS models (#800)

.gitignore

@@ -93,3 +93,5 @@ sr-data
 vits-icefall-*
 sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12
 spoken-language-identification-test-wavs
+my-release-key*
+vits-zh-hf-fanchen-C

android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt

@@ -158,6 +158,7 @@ class MainActivity : AppCompatActivity() {
         var ruleFars: String?
         var lexicon: String?
         var dataDir: String?
+        var dictDir: String?
         var assets: AssetManager? = application.assets
 
         // The purpose of such a design is to make the CI test easier
@@ -169,6 +170,7 @@ class MainActivity : AppCompatActivity() {
         ruleFars = null
         lexicon = null
         dataDir = null
+        dictDir = null
 
         // Example 1:
         // modelDir = "vits-vctk"
@@ -191,21 +193,36 @@ class MainActivity : AppCompatActivity() {
         // lexicon = "lexicon.txt"
 
         // Example 4:
+        // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#csukuangfj-vits-zh-hf-fanchen-c-chinese-187-speakers
+        // modelDir = "vits-zh-hf-fanchen-C"
+        // modelName = "vits-zh-hf-fanchen-C.onnx"
+        // lexicon = "lexicon.txt"
+        // dictDir = "vits-zh-hf-fanchen-C/dict"
+
+        // Example 5:
         // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2
         // modelDir = "vits-coqui-de-css10"
         // modelName = "model.onnx"
-        // lang = "deu"
 
         if (dataDir != null) {
-            val newDir = copyDataDir(modelDir)
+            val newDir = copyDataDir(modelDir!!)
             modelDir = newDir + "/" + modelDir
             dataDir = newDir + "/" + dataDir
             assets = null
         }
 
+        if (dictDir != null) {
+            val newDir = copyDataDir( modelDir!!)
+            modelDir = newDir + "/" + modelDir
+            dictDir = modelDir + "/" + "dict"
+            ruleFsts = "$modelDir/phone.fst,$modelDir/date.fst,$modelDir/number.fst"
+            assets = null
+        }
+
         val config = getOfflineTtsConfig(
             modelDir = modelDir!!, modelName = modelName!!, lexicon = lexicon ?: "",
             dataDir = dataDir ?: "",
+            dictDir = dictDir ?: "",
             ruleFsts = ruleFsts ?: "",
             ruleFars = ruleFars ?: "",
         )!!

android/SherpaOnnxTts/app/src/main/java/com/k2fsa/sherpa/onnx/Tts.kt

@@ -8,6 +8,7 @@ data class OfflineTtsVitsModelConfig(
     var lexicon: String = "",
     var tokens: String,
     var dataDir: String = "",
+    var dictDir: String = "",
     var noiseScale: Float = 0.667f,
     var noiseScaleW: Float = 0.8f,
     var lengthScale: Float = 1.0f,
@@ -49,7 +50,7 @@ class OfflineTts(
 
     init {
         if (assetManager != null) {
-            ptr = new(assetManager, config)
+            ptr = newFromAsset(assetManager, config)
         } else {
             ptr = newFromFile(config)
         }
@@ -87,7 +88,7 @@ class OfflineTts(
     fun allocate(assetManager: AssetManager? = null) {
         if (ptr == 0L) {
             if (assetManager != null) {
-                ptr = new(assetManager, config)
+                ptr = newFromAsset(assetManager, config)
             } else {
                 ptr = newFromFile(config)
             }
@@ -105,7 +106,7 @@ class OfflineTts(
         delete(ptr)
     }
 
-    private external fun new(
+    private external fun newFromAsset(
         assetManager: AssetManager,
         config: OfflineTtsConfig,
     ): Long
@@ -152,6 +153,7 @@ fun getOfflineTtsConfig(
     modelName: String,
     lexicon: String,
     dataDir: String,
+    dictDir: String,
     ruleFsts: String,
     ruleFars: String
 ): OfflineTtsConfig? {
@@ -161,7 +163,8 @@ fun getOfflineTtsConfig(
                 model = "$modelDir/$modelName",
                 lexicon = "$modelDir/$lexicon",
                 tokens = "$modelDir/tokens.txt",
-                dataDir = "$dataDir"
+                dataDir = dataDir,
+                dictDir = dictDir,
             ),
             numThreads = 2,
             debug = true,

android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt

@@ -42,6 +42,7 @@ object TtsEngine {
     private var ruleFars: String? = null
     private var lexicon: String? = null
     private var dataDir: String? = null
+    private var dictDir: String? = null
     private var assets: AssetManager? = null
 
     init {
@@ -54,6 +55,7 @@ object TtsEngine {
         ruleFars = null
         lexicon = null
         dataDir = null
+        dictDir = null
         lang = null
 
         // Please enable one and only one of the examples below
@@ -83,6 +85,14 @@ object TtsEngine {
         // lang = "zho"
 
         // Example 4:
+        // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#csukuangfj-vits-zh-hf-fanchen-c-chinese-187-speakers
+        // modelDir = "vits-zh-hf-fanchen-C"
+        // modelName = "vits-zh-hf-fanchen-C.onnx"
+        // lexicon = "lexicon.txt"
+        // dictDir = "vits-zh-hf-fanchen-C/dict"
+        // lang = "zho"
+
+        // Example 5:
         // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2
         // This model does not need lexicon or dataDir
         // modelDir = "vits-coqui-de-css10"
@@ -108,9 +118,18 @@ object TtsEngine {
             assets = null
         }
 
+        if (dictDir != null) {
+            val newDir = copyDataDir(context, modelDir!!)
+            modelDir = newDir + "/" + modelDir
+            dictDir = modelDir + "/" + "dict"
+            ruleFsts = "$modelDir/phone.fst,$modelDir/date.fst,$modelDir/number.fst"
+            assets = null
+        }
+
         val config = getOfflineTtsConfig(
             modelDir = modelDir!!, modelName = modelName!!, lexicon = lexicon ?: "",
             dataDir = dataDir ?: "",
+            dictDir = dictDir ?: "",
             ruleFsts = ruleFsts ?: "",
             ruleFars = ruleFars ?: ""
         )!!

build-android-arm64-v8a.sh

@@ -47,7 +47,7 @@ onnxruntime_version=1.17.1
 if [ ! -f $onnxruntime_version/jni/arm64-v8a/libonnxruntime.so ]; then
   mkdir -p $onnxruntime_version
   pushd $onnxruntime_version
-  wget -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
+  wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
   unzip onnxruntime-android-${onnxruntime_version}.zip
   rm onnxruntime-android-${onnxruntime_version}.zip
   popd

build-android-armv7-eabi.sh

@@ -48,7 +48,7 @@ onnxruntime_version=1.17.1
 if [ ! -f $onnxruntime_version/jni/armeabi-v7a/libonnxruntime.so ]; then
   mkdir -p $onnxruntime_version
   pushd $onnxruntime_version
-  wget -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
+  wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
   unzip onnxruntime-android-${onnxruntime_version}.zip
   rm onnxruntime-android-${onnxruntime_version}.zip
   popd

build-android-x86-64.sh

@@ -48,7 +48,7 @@ onnxruntime_version=1.17.1
 if [ ! -f $onnxruntime_version/jni/x86_64/libonnxruntime.so ]; then
   mkdir -p $onnxruntime_version
   pushd $onnxruntime_version
-  wget -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
+  wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
   unzip onnxruntime-android-${onnxruntime_version}.zip
   rm onnxruntime-android-${onnxruntime_version}.zip
   popd

build-android-x86.sh

@@ -48,7 +48,7 @@ onnxruntime_version=1.17.1
 if [ ! -f $onnxruntime_version/jni/x86/libonnxruntime.so ]; then
   mkdir -p $onnxruntime_version
   pushd $onnxruntime_version
-  wget -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
+  wget -c -q https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${onnxruntime_version}/onnxruntime-android-${onnxruntime_version}.zip
   unzip onnxruntime-android-${onnxruntime_version}.zip
   rm onnxruntime-android-${onnxruntime_version}.zip
   popd

scripts/apk/build-apk-tts-engine.sh.in

@@ -61,6 +61,11 @@ sed -i.bak s/"lang = null"/"lang = \"$lang_iso_639_3\""/ ./TtsEngine.kt
   sed -i.bak s%"ruleFsts = null"%"ruleFars = \"$rule_fars\""% ./TtsEngine.kt
 {% endif %}
 
+{% if tts_model.dict_dir %}
+  dict_dir={{ tts_model.dict_dir }}
+  sed -i.bak s%"dictDir = null"%"dictDir = \"$dict_dir\""% ./TtsEngine.kt
+{% endif %}
+
 {% if tts_model.data_dir %}
   data_dir={{ tts_model.data_dir }}
   sed -i.bak s%"dataDir = null"%"dataDir = \"$data_dir\""% ./TtsEngine.kt

scripts/apk/build-apk-tts.sh.in

@@ -59,6 +59,11 @@ sed -i.bak s/"modelName = null"/"modelName = \"$model_name\""/ ./MainActivity.kt
   sed -i.bak s%"ruleFsts = null"%"ruleFars = \"$rule_fars\""% ./MainActivity.kt
 {% endif %}
 
+{% if tts_model.dict_dir %}
+  dict_dir={{ tts_model.dict_dir }}
+  sed -i.bak s%"dictDir = null"%"dictDir = \"$dict_dir\""% ./MainActivity.kt
+{% endif %}
+
 {% if tts_model.data_dir %}
   data_dir={{ tts_model.data_dir }}
   sed -i.bak s%"dataDir = null"%"dataDir = \"$data_dir\""% ./MainActivity.kt

scripts/apk/generate-tts-apk-script.py

@@ -35,6 +35,7 @@ class TtsModel:
     rule_fsts: Optional[List[str]] = None
     rule_fars: Optional[List[str]] = None
     data_dir: Optional[str] = None
+    dict_dir: Optional[str] = None
     is_char: bool = False
     lang_iso_639_3: str = ""
 
@@ -326,8 +327,14 @@ def get_vits_models() -> List[TtsModel]:
     rule_fsts = ["phone.fst", "date.fst", "number.fst", "new_heteronym.fst"]
     for m in chinese_models:
         s = [f"{m.model_dir}/{r}" for r in rule_fsts]
+        if "vits-zh-hf" in m.model_dir:
+            s = s[:-1]
+            m.dict_dir = m.model_dir + "/dict"
+
         m.rule_fsts = ",".join(s)
-        m.rule_fars = f"{m.model_dir}/rule.far"
+
+        if "vits-zh-hf" not in m.model_dir:
+            m.rule_fars = f"{m.model_dir}/rule.far"
 
     all_models = chinese_models + [
         TtsModel(

sherpa-onnx/csrc/audio-tagging-model-config.cc

@@ -32,7 +32,7 @@ bool AudioTaggingModelConfig::Validate() const {
   }
 
   if (!ced.empty() && !FileExists(ced)) {
-    SHERPA_ONNX_LOGE("CED model file %s does not exist", ced.c_str());
+    SHERPA_ONNX_LOGE("CED model file '%s' does not exist", ced.c_str());
     return false;
   }
 

sherpa-onnx/csrc/audio-tagging.cc

@@ -48,7 +48,7 @@ bool AudioTaggingConfig::Validate() const {
   }
 
   if (!FileExists(labels)) {
-    SHERPA_ONNX_LOGE("--labels %s does not exist", labels.c_str());
+    SHERPA_ONNX_LOGE("--labels '%s' does not exist", labels.c_str());
     return false;
   }
 

sherpa-onnx/csrc/file-utils.cc

@@ -7,7 +7,7 @@
 #include <fstream>
 #include <string>
 
-#include "sherpa-onnx/csrc/log.h"
+#include "sherpa-onnx/csrc/macros.h"
 
 namespace sherpa_onnx {
 
@@ -17,7 +17,7 @@ bool FileExists(const std::string &filename) {
 
 void AssertFileExists(const std::string &filename) {
   if (!FileExists(filename)) {
-    SHERPA_ONNX_LOG(FATAL) << filename << " does not exist!";
+    SHERPA_ONNX_LOGE("filename '%s' does not exist", filename.c_str());
     exit(-1);
   }
 }

sherpa-onnx/csrc/jieba-lexicon.cc

@@ -146,6 +146,14 @@ class JiebaLexicon::Impl {
       if (token2id_.count(p.first) && !token2id_.count(p.second)) {
         token2id_[p.second] = token2id_[p.first];
       }
+
+      if (!token2id_.count(p.first) && token2id_.count(p.second)) {
+        token2id_[p.first] = token2id_[p.second];
+      }
+    }
+
+    if (!token2id_.count("、") && token2id_.count("，")) {
+      token2id_["、"] = token2id_["，"];
     }
   }
 

sherpa-onnx/csrc/keyword-spotter.cc

@@ -101,7 +101,8 @@ bool KeywordSpotterConfig::Validate() const {
   // Solution: take keyword_file variable is directly
   // parsed as a string of keywords
   if (!std::ifstream(keywords_file.c_str()).good()) {
-    SHERPA_ONNX_LOGE("Keywords file %s does not exist.", keywords_file.c_str());
+    SHERPA_ONNX_LOGE("Keywords file '%s' does not exist.",
+                     keywords_file.c_str());
     return false;
   }
 #endif

sherpa-onnx/csrc/offline-ctc-fst-decoder-config.cc

@@ -34,7 +34,7 @@ void OfflineCtcFstDecoderConfig::Register(ParseOptions *po) {
 
 bool OfflineCtcFstDecoderConfig::Validate() const {
   if (!graph.empty() && !FileExists(graph)) {
-    SHERPA_ONNX_LOGE("graph: %s does not exist", graph.c_str());
+    SHERPA_ONNX_LOGE("graph: '%s' does not exist", graph.c_str());
     return false;
   }
   return true;

sherpa-onnx/csrc/offline-lm-config.cc

@@ -22,7 +22,7 @@ void OfflineLMConfig::Register(ParseOptions *po) {
 
 bool OfflineLMConfig::Validate() const {
   if (!FileExists(model)) {
-    SHERPA_ONNX_LOGE("%s does not exist", model.c_str());
+    SHERPA_ONNX_LOGE("'%s' does not exist", model.c_str());
     return false;
   }
 

sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model-config.cc

@@ -16,7 +16,7 @@ void OfflineNemoEncDecCtcModelConfig::Register(ParseOptions *po) {
 
 bool OfflineNemoEncDecCtcModelConfig::Validate() const {
   if (!FileExists(model)) {
-    SHERPA_ONNX_LOGE("NeMo model: %s does not exist", model.c_str());
+    SHERPA_ONNX_LOGE("NeMo model: '%s' does not exist", model.c_str());
     return false;
   }
 

sherpa-onnx/csrc/offline-paraformer-model-config.cc

@@ -15,7 +15,7 @@ void OfflineParaformerModelConfig::Register(ParseOptions *po) {
 
 bool OfflineParaformerModelConfig::Validate() const {
   if (!FileExists(model)) {
-    SHERPA_ONNX_LOGE("Paraformer model %s does not exist", model.c_str());
+    SHERPA_ONNX_LOGE("Paraformer model '%s' does not exist", model.c_str());
     return false;
   }
 

sherpa-onnx/csrc/offline-transducer-model-config.cc

@@ -18,19 +18,19 @@ void OfflineTransducerModelConfig::Register(ParseOptions *po) {
 
 bool OfflineTransducerModelConfig::Validate() const {
   if (!FileExists(encoder_filename)) {
-    SHERPA_ONNX_LOGE("transducer encoder: %s does not exist",
+    SHERPA_ONNX_LOGE("transducer encoder: '%s' does not exist",
                      encoder_filename.c_str());
     return false;
   }
 
   if (!FileExists(decoder_filename)) {
-    SHERPA_ONNX_LOGE("transducer decoder: %s does not exist",
+    SHERPA_ONNX_LOGE("transducer decoder: '%s' does not exist",
                      decoder_filename.c_str());
     return false;
   }
 
   if (!FileExists(joiner_filename)) {
-    SHERPA_ONNX_LOGE("transducer joiner: %s does not exist",
+    SHERPA_ONNX_LOGE("transducer joiner: '%s' does not exist",
                      joiner_filename.c_str());
     return false;
   }