Add Kotlin and Java API for ZipVoice models

.github/workflows/jni.yaml

@@ -8,6 +8,7 @@ on:
       - '.github/workflows/jni.yaml'
       - 'cmake/**'
       - 'kotlin-api-examples/**'
+      - 'sherpa-onnx/kotlin-api/**'
       - 'sherpa-onnx/csrc/*'
       - 'sherpa-onnx/jni/*'
 

.github/workflows/run-java-test.yaml

@@ -341,23 +341,26 @@ jobs:
         run: |
           cd ./java-api-examples
 
-          ./run-pocket-tts.sh
-          ./run-supertonic-tts.sh
-          ./run-non-streaming-tts-kitten-en.sh
-          ./run-non-streaming-tts-kokoro-zh-en.sh
-          ./run-non-streaming-tts-kokoro-en.sh
-          ./run-non-streaming-tts-matcha-zh.sh
+           ./run-pocket-tts.sh
+           ./run-zipvoice-tts.sh
+           ./run-supertonic-tts.sh
+           ./run-non-streaming-tts-kitten-en.sh
+           ./run-non-streaming-tts-kokoro-zh-en.sh
+           ./run-non-streaming-tts-kokoro-en.sh
+           ./run-non-streaming-tts-matcha-zh.sh
           ./run-non-streaming-tts-matcha-en.sh
           ls -lh
 
-          rm -rf sherpa-onnx-pocket-tts-*
-          rm -rf sherpa-onnx-supertonic-tts-*
-          rm -rf kitten-nano-en-*
-          rm -rf kokoro-multi-*
-          rm -rf kokoro-en-*
+           rm -rf sherpa-onnx-pocket-tts-*
+           rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia
+           rm -rf sherpa-onnx-supertonic-tts-*
+           rm -rf kitten-nano-en-*
+           rm -rf kokoro-multi-*
+           rm -rf kokoro-en-*
 
-          rm -rf matcha-icefall-*
-          rm vocos-22khz-univ.onnx
+           rm -rf matcha-icefall-*
+           rm vocos-22khz-univ.onnx
+           rm vocos_24khz.onnx
 
           ./run-non-streaming-tts-piper-en.sh
           rm -rf vits-piper-*

java-api-examples/README.md

@@ -76,6 +76,7 @@ downstream ASR and `dpdfnet2_48khz_hr.onnx` for 48 kHz enhancement output.
 ./run-non-streaming-tts-piper-en.sh
 ./run-non-streaming-tts-vits-zh.sh
 ./run-pocket-tts.sh
+./run-zipvoice-tts.sh
 ```
 
 ## Non-Streaming text-to-speech (Playback the audio as it is being generated)

java-api-examples/ZipVoiceTts.java

@@ -0,0 +1,77 @@
+// Copyright 2026 Xiaomi Corporation
+
+// This file shows how to use a ZipVoice Chinese/English model
+// for zero-shot text to speech.
+import com.k2fsa.sherpa.onnx.*;
+import java.util.HashMap;
+import java.util.Map;
+
+public class ZipVoiceTts {
+  public static void main(String[] args) {
+    LibraryUtils.enableDebug();
+    // please visit
+    // https://k2-fsa.github.io/sherpa/onnx/tts/zipvoice.html
+    // to download model files
+    String modelDir = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia";
+    String referenceAudioFilename = modelDir + "/test_wavs/leijun-1.wav";
+    String text = "小米的价值观是真诚, 热爱. 真诚，就是不欺人也不自欺. 热爱, 就是全心投入并享受其中.";
+    String referenceText = "那还是三十六年前, 一九八七年. 我呢考上了武汉大学的计算机系.";
+
+    OfflineTtsZipVoiceModelConfig zipvoiceModelConfig =
+        OfflineTtsZipVoiceModelConfig.builder()
+            .setTokens(modelDir + "/tokens.txt")
+            .setEncoder(modelDir + "/encoder.int8.onnx")
+            .setDecoder(modelDir + "/decoder.int8.onnx")
+            .setVocoder("./vocos_24khz.onnx")
+            .setDataDir(modelDir + "/espeak-ng-data")
+            .setLexicon(modelDir + "/lexicon.txt")
+            .build();
+
+    OfflineTtsModelConfig modelConfig =
+        OfflineTtsModelConfig.builder()
+            .setZipvoice(zipvoiceModelConfig)
+            .setNumThreads(2)
+            .setDebug(false)
+            .build();
+
+    OfflineTtsConfig config = OfflineTtsConfig.builder().setModel(modelConfig).build();
+    OfflineTts tts = new OfflineTts(config);
+
+    WaveReader reader = new WaveReader(referenceAudioFilename);
+
+    GenerationConfig genConfig = new GenerationConfig();
+    genConfig.setReferenceAudio(reader.getSamples());
+    genConfig.setReferenceSampleRate(reader.getSampleRate());
+    genConfig.setReferenceText(referenceText);
+    genConfig.setNumSteps(4);
+
+    Map<String, String> extra = new HashMap<>();
+    extra.put("min_char_in_sentence", "10");
+    genConfig.setExtra(extra);
+
+    long start = System.currentTimeMillis();
+    GeneratedAudio audio =
+        tts.generateWithConfigAndCallback(
+            text,
+            genConfig,
+            samples -> {
+              System.out.println("callback got called with " + samples.length + " samples");
+              return 1;
+            });
+    long stop = System.currentTimeMillis();
+
+    float timeElapsedSeconds = (stop - start) / 1000.0f;
+    float audioDuration = audio.getSamples().length / (float) audio.getSampleRate();
+    float realTimeFactor = timeElapsedSeconds / audioDuration;
+
+    String waveFilename = "generated-zipvoice-zh-en-java.wav";
+    audio.save(waveFilename);
+    System.out.printf("-- elapsed : %.3f seconds\n", timeElapsedSeconds);
+    System.out.printf("-- audio duration: %.3f seconds\n", audioDuration);
+    System.out.printf("-- real-time factor (RTF): %.3f\n", realTimeFactor);
+    System.out.printf("-- text: %s\n", text);
+    System.out.printf("-- Saved to %s\n", waveFilename);
+
+    tts.release();
+  }
+}

java-api-examples/run-zipvoice-tts.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib  && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
+  mkdir -p ../build
+  pushd ../build
+  cmake \
+    -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
+    -DSHERPA_ONNX_ENABLE_TESTS=OFF \
+    -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+    -DBUILD_SHARED_LIBS=ON \
+    -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+    -DSHERPA_ONNX_ENABLE_JNI=ON \
+    ..
+
+  make -j4
+  ls -lh lib
+  popd
+fi
+
+if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
+  pushd ../sherpa-onnx/java-api
+  make
+  popd
+fi
+
+# please visit
+# https://k2-fsa.github.io/sherpa/onnx/tts/zipvoice.html
+# to download more models
+if [ ! -f ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2
+  tar xf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2
+  rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2
+fi
+
+if [ ! -f ./vocos_24khz.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx
+fi
+
+java \
+  -Djava.library.path=$PWD/../build/lib \
+  -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
+  ZipVoiceTts.java

kotlin-api-examples/run.sh

@@ -57,6 +57,30 @@ function testPocketTts() {
   java -Djava.library.path=../build/lib -jar $out_filename
 }
 
+function testZipVoiceTts() {
+  if [ ! -f ./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia/encoder.int8.onnx ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2
+    tar xvf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2
+    rm sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2
+  fi
+
+  if [ ! -f ./vocos_24khz.onnx ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx
+  fi
+
+  out_filename=test_zipvoice_tts.jar
+  kotlinc-jvm -include-runtime -d $out_filename \
+    test_zipvoice_tts.kt \
+    Tts.kt \
+    WaveReader.kt \
+    faked-asset-manager.kt \
+    faked-log.kt
+
+  ls -lh $out_filename
+
+  java -Djava.library.path=../build/lib -jar $out_filename
+}
+
 function testSupertonicTts() {
   if [ ! -f ./sherpa-onnx-supertonic-tts-int8-2026-03-06/duration_predictor.int8.onnx ]; then
     curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-supertonic-tts-int8-2026-03-06.tar.bz2
@@ -717,6 +741,7 @@ testVersion
 testOfflineMoonshineAsrV2
 testOfflineFireRedAsrCtc
 testPocketTts
+testZipVoiceTts
 testSupertonicTts
 testOfflineFunAsrNano
 testOfflineMedAsrCtc

kotlin-api-examples/test_zipvoice_tts.kt

@@ -0,0 +1,50 @@
+package com.k2fsa.sherpa.onnx
+
+fun main() {
+  testZipVoiceTts()
+}
+
+fun testZipVoiceTts() {
+  val modelDir = "./sherpa-onnx-zipvoice-distill-int8-zh-en-emilia"
+  val referenceAudioFilename = "$modelDir/test_wavs/leijun-1.wav"
+  val wave = WaveReader.readWave(filename = referenceAudioFilename)
+
+  val config = OfflineTtsConfig(
+    model = OfflineTtsModelConfig(
+      zipvoice = OfflineTtsZipVoiceModelConfig(
+        tokens = "$modelDir/tokens.txt",
+        encoder = "$modelDir/encoder.int8.onnx",
+        decoder = "$modelDir/decoder.int8.onnx",
+        vocoder = "./vocos_24khz.onnx",
+        dataDir = "$modelDir/espeak-ng-data",
+        lexicon = "$modelDir/lexicon.txt",
+      ),
+      numThreads = 2,
+      debug = false,
+    ),
+  )
+
+  val tts = OfflineTts(config = config)
+  val text = "小米的价值观是真诚, 热爱. 真诚，就是不欺人也不自欺. 热爱, 就是全心投入并享受其中."
+  val referenceText = "那还是三十六年前, 一九八七年. 我呢考上了武汉大学的计算机系."
+  val genConfig = GenerationConfig(
+    referenceAudio = wave.samples,
+    referenceSampleRate = wave.sampleRate,
+    referenceText = referenceText,
+    numSteps = 4,
+    extra = mapOf("min_char_in_sentence" to "10"),
+  )
+
+  val audio = tts.generateWithConfigAndCallback(text = text, config = genConfig, callback = ::callback)
+  audio.save(filename = "test-zipvoice-zh-en.wav")
+  tts.release()
+  println("Saved to test-zipvoice-zh-en.wav")
+}
+
+fun callback(samples: FloatArray): Int {
+  println("callback got called with ${samples.size} samples")
+
+  // 1 means to continue
+  // 0 means to stop
+  return 1
+}

sherpa-onnx/java-api/Makefile

@@ -57,6 +57,7 @@ java_files += OfflineTtsKittenModelConfig.java
 java_files += OfflineTtsPocketModelConfig.java
 java_files += OfflineTtsSupertonicModelConfig.java
 java_files += OfflineTtsKokoroModelConfig.java
+java_files += OfflineTtsZipVoiceModelConfig.java
 java_files += OfflineTtsMatchaModelConfig.java
 java_files += OfflineTtsVitsModelConfig.java
 java_files += OfflineTtsModelConfig.java

sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineTtsModelConfig.java

@@ -6,6 +6,7 @@ public class OfflineTtsModelConfig {
     private final OfflineTtsVitsModelConfig vits;
     private final OfflineTtsMatchaModelConfig matcha;
     private final OfflineTtsKokoroModelConfig kokoro;
+    private final OfflineTtsZipVoiceModelConfig zipvoice;
     private final OfflineTtsKittenModelConfig kitten;
     private final OfflineTtsPocketModelConfig pocket;
     private final OfflineTtsSupertonicModelConfig supertonic;
@@ -17,6 +18,7 @@ private OfflineTtsModelConfig(Builder builder) {
         this.vits = builder.vits;
         this.matcha = builder.matcha;
         this.kokoro = builder.kokoro;
+        this.zipvoice = builder.zipvoice;
         this.kitten = builder.kitten;
         this.pocket = builder.pocket;
         this.supertonic = builder.supertonic;
@@ -41,6 +43,10 @@ public OfflineTtsKokoroModelConfig getKokoro() {
         return kokoro;
     }
 
+    public OfflineTtsZipVoiceModelConfig getZipvoice() {
+        return zipvoice;
+    }
+
     public OfflineTtsKittenModelConfig getKitten() {
         return kitten;
     }
@@ -57,6 +63,7 @@ public static class Builder {
         private OfflineTtsVitsModelConfig vits = OfflineTtsVitsModelConfig.builder().build();
         private OfflineTtsMatchaModelConfig matcha = OfflineTtsMatchaModelConfig.builder().build();
         private OfflineTtsKokoroModelConfig kokoro = OfflineTtsKokoroModelConfig.builder().build();
+        private OfflineTtsZipVoiceModelConfig zipvoice = OfflineTtsZipVoiceModelConfig.builder().build();
         private OfflineTtsKittenModelConfig kitten = OfflineTtsKittenModelConfig.builder().build();
         private OfflineTtsPocketModelConfig pocket = OfflineTtsPocketModelConfig.builder().build();
         private OfflineTtsSupertonicModelConfig supertonic = OfflineTtsSupertonicModelConfig.builder().build();
@@ -83,6 +90,11 @@ public Builder setKokoro(OfflineTtsKokoroModelConfig kokoro) {
             return this;
         }
 
+        public Builder setZipvoice(OfflineTtsZipVoiceModelConfig zipvoice) {
+            this.zipvoice = zipvoice;
+            return this;
+        }
+
         public Builder setKitten(OfflineTtsKittenModelConfig kitten) {
             this.kitten = kitten;
             return this;