From 5faa6a95938b11274ce61dbb5c58799d9f1a0db6 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 17:01:09 +0800 Subject: [PATCH 01/10] Fix C# API --- scripts/dotnet/OfflineFunAsrNanoModel.cs | 10 ++++++++++ scripts/dotnet/OfflineWhisperModelConfig.cs | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/dotnet/OfflineFunAsrNanoModel.cs b/scripts/dotnet/OfflineFunAsrNanoModel.cs index 3b1127a5f2..9dd8f4646a 100644 --- a/scripts/dotnet/OfflineFunAsrNanoModel.cs +++ b/scripts/dotnet/OfflineFunAsrNanoModel.cs @@ -20,6 +20,9 @@ public OfflineFunAsrNanoModelConfig() Temperature = 1e-6F; TopP = 0.8F; Seed = 42; + Language = ""; + Itn = 0; + Hotwords = ""; } [MarshalAs(UnmanagedType.LPStr)] @@ -44,5 +47,12 @@ public OfflineFunAsrNanoModelConfig() public float Temperature; public float TopP; public int Seed; + [MarshalAs(UnmanagedType.LPStr)] + public string Language; + + public int Itn; + + [MarshalAs(UnmanagedType.LPStr)] + public string Hotwords; } } diff --git a/scripts/dotnet/OfflineWhisperModelConfig.cs b/scripts/dotnet/OfflineWhisperModelConfig.cs index 16b6387c1a..0b93b74a72 100644 --- a/scripts/dotnet/OfflineWhisperModelConfig.cs +++ b/scripts/dotnet/OfflineWhisperModelConfig.cs @@ -14,6 +14,8 @@ public OfflineWhisperModelConfig() Language = ""; Task = "transcribe"; TailPaddings = -1; + EnableTokenTimestamps = 0; + EnableSegmentTimestamps = 0; } [MarshalAs(UnmanagedType.LPStr)] public string Encoder; @@ -28,6 +30,8 @@ public OfflineWhisperModelConfig() public string Task; public int TailPaddings; + public int EnableTokenTimestamps; + public int EnableSegmentTimestamps; } -} \ No newline at end of file +} From 4bcae11389187cd4e0fdfa71a4bf0f5593dac31f Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 17:21:31 +0800 Subject: [PATCH 02/10] Begin to add C# API for PocketTTS --- scripts/dotnet/OfflineTts.cs | 19 ++++ scripts/dotnet/OfflineTtsGenerationConfig.cs | 96 +++++++++++++++++++ scripts/dotnet/OfflineTtsPocketModelConfig.cs | 44 +++++++++ 3 files changed, 159 insertions(+) create mode 100644 scripts/dotnet/OfflineTtsGenerationConfig.cs create mode 100644 scripts/dotnet/OfflineTtsPocketModelConfig.cs diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index 334ba2287b..e4bb248790 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -8,6 +8,8 @@ namespace SherpaOnnx // IntPtr is actually a `const float*` from C++ public delegate int OfflineTtsCallback(IntPtr samples, int n); public delegate int OfflineTtsCallbackProgress(IntPtr samples, int n, float progress); + public delegate int OfflineTtsCallbackProgressWithArg(IntPtr samples, int n, float progress, IntPtr arg); + public class OfflineTts : IDisposable { @@ -47,6 +49,20 @@ public OfflineTtsGeneratedAudio GenerateWithCallbackProgress(String text, float return new OfflineTtsGeneratedAudio(p); } + public OfflineTtsGeneratedAudio GenerateWithConfig(String text, OfflineTtsGenerationConfig confi, OfflineTtsCallbackProgressWithArg callback) + { + byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); + utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator + + OfflineTtsGenerationConfig.NativeStruct nativeConfig = config.ToNative(); + + + IntPtr p = SherpaOnnxOfflineTtsGenerateWithConfig(_handle.Handle, utf8BytesWithNull, ref nativeConfig, callback, IntPtr.Zero); + return new OfflineTtsGeneratedAudio(p); + } + public void Dispose() { Cleanup(); @@ -106,5 +122,8 @@ public int NumSpeakers [DllImport(Dll.Filename, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr SherpaOnnxOfflineTtsGenerateWithProgressCallback(IntPtr handle, [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.I1)] byte[] utf8Text, int sid, float speed, OfflineTtsCallbackProgress callback); + + [DllImport(Dll.Filename, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr SherpaOnnxOfflineTtsGenerateWithConfig(IntPtr handle, [MarshalAs(UnmanagedType.LPArray, ArraySubType = UnmanagedType.I1)] byte[] utf8Text, ref OfflineTtsGenerationConfig.NativeStruct config, OfflineTtsCallbackProgressWithArg callback, IntPtr arg); } } diff --git a/scripts/dotnet/OfflineTtsGenerationConfig.cs b/scripts/dotnet/OfflineTtsGenerationConfig.cs new file mode 100644 index 0000000000..a3d54645a2 --- /dev/null +++ b/scripts/dotnet/OfflineTtsGenerationConfig.cs @@ -0,0 +1,96 @@ +/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) + +using System; +using System.Collections; +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + public class OfflineTtsGenerationConfig + { + public float SilenceScale; + public float Speed; + public int Sid; + + public float[] ReferenceAudio; + public int ReferenceSampleRate; + public string ReferenceText; + public int NumSteps; + + /// + /// Extra attributes serialized as JSON manually + /// + public Hashtable Extra; + + public OfflineTtsGenerationConfig() + { + SilenceScale = 1.0f; + Speed = 1.0f; + Sid = 0; + ReferenceAudio = null; + ReferenceSampleRate = 16000; + ReferenceText = ""; + NumSteps = 0; + Extra = new Hashtable(); + } + + internal NativeStruct ToNative() + { + NativeStruct native = new NativeStruct(); + native.SilenceScale = SilenceScale; + native.Speed = Speed; + native.Sid = Sid; + native.ReferenceAudio = ReferenceAudio; + native.ReferenceAudioLen = (ReferenceAudio != null) ? ReferenceAudio.Length : 0; + native.ReferenceSampleRate = ReferenceSampleRate; + native.ReferenceText = (ReferenceText != null) ? ReferenceText : ""; + + native.NumSteps = NumSteps; + + // Simple JSON string generation + native.Extra = "{}"; + if (Extra != null && Extra.Count > 0) + { + string json = "{"; + bool first = true; + foreach (DictionaryEntry kv in Extra) + { + if (!first) json += ","; + string key = kv.Key.ToString(); + string val; + if (kv.Value is string) + val = "\"" + kv.Value.ToString() + "\""; + else + val = kv.Value.ToString(); + json += "\"" + key + "\":" + val; + first = false; + } + json += "}"; + native.Extra = json; + } + + return native; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct NativeStruct + { + public float SilenceScale; + public float Speed; + public int Sid; + + public float[] ReferenceAudio; + public int ReferenceAudioLen; + public int ReferenceSampleRate; + + [MarshalAs(UnmanagedType.LPStr)] + public string ReferenceText; + + public int NumSteps; + + [MarshalAs(UnmanagedType.LPStr)] + public string Extra; + } + } +} + diff --git a/scripts/dotnet/OfflineTtsPocketModelConfig.cs b/scripts/dotnet/OfflineTtsPocketModelConfig.cs new file mode 100644 index 0000000000..5a748e2473 --- /dev/null +++ b/scripts/dotnet/OfflineTtsPocketModelConfig.cs @@ -0,0 +1,44 @@ +/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) + +using System.Runtime.InteropServices; + +namespace SherpaOnnx +{ + [StructLayout(LayoutKind.Sequential)] + public struct OfflineTtsPocketModelConfig + { + // Default constructor for convenience + public OfflineTtsPocketModelConfig() + { + LmFlow = ""; + LmMain = ""; + Encoder = ""; + Decoder = ""; + TextConditioner = ""; + VocabJson = ""; + TokenScoresJson = ""; + } + + [MarshalAs(UnmanagedType.LPStr)] + public string LmFlow; + + [MarshalAs(UnmanagedType.LPStr)] + public string LmMain; + + [MarshalAs(UnmanagedType.LPStr)] + public string Encoder; + + [MarshalAs(UnmanagedType.LPStr)] + public string Decoder; + + [MarshalAs(UnmanagedType.LPStr)] + public string TextConditioner; + + [MarshalAs(UnmanagedType.LPStr)] + public string VocabJson; + + [MarshalAs(UnmanagedType.LPStr)] + public string TokenScoresJson; + } +} + From f554e77415f17dc661565848ba4c07f5525ff835 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 17:22:27 +0800 Subject: [PATCH 03/10] Begin to add demo for pocket tts --- dotnet-examples/pocket-tts-zero-shot/Program.cs | 2 ++ .../pocket-tts-zero-shot/pocket-tts-zero-shot.csproj | 11 +++++++++++ dotnet-examples/sherpa-onnx.sln | 6 ++++++ 3 files changed, 19 insertions(+) create mode 100644 dotnet-examples/pocket-tts-zero-shot/Program.cs create mode 100644 dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj diff --git a/dotnet-examples/pocket-tts-zero-shot/Program.cs b/dotnet-examples/pocket-tts-zero-shot/Program.cs new file mode 100644 index 0000000000..83fa4f4d5f --- /dev/null +++ b/dotnet-examples/pocket-tts-zero-shot/Program.cs @@ -0,0 +1,2 @@ +// See https://aka.ms/new-console-template for more information +Console.WriteLine("Hello, World!"); diff --git a/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj b/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj new file mode 100644 index 0000000000..a8f40655a0 --- /dev/null +++ b/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj @@ -0,0 +1,11 @@ + + + + Exe + net8.0 + pocket_tts_zero_shot + enable + enable + + + diff --git a/dotnet-examples/sherpa-onnx.sln b/dotnet-examples/sherpa-onnx.sln index f58e121c3d..0c0a7a3294 100644 --- a/dotnet-examples/sherpa-onnx.sln +++ b/dotnet-examples/sherpa-onnx.sln @@ -51,6 +51,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "non-streaming-funasr-nano-d EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "vad-non-streaming-funasr-nano", "vad-non-streaming-funasr-nano\vad-non-streaming-funasr-nano.csproj", "{32C8C12B-D7DB-455E-B35C-945A745520CC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "pocket-tts-zero-shot", "pocket-tts-zero-shot\pocket-tts-zero-shot.csproj", "{9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -153,6 +155,10 @@ Global {32C8C12B-D7DB-455E-B35C-945A745520CC}.Debug|Any CPU.Build.0 = Debug|Any CPU {32C8C12B-D7DB-455E-B35C-945A745520CC}.Release|Any CPU.ActiveCfg = Release|Any CPU {32C8C12B-D7DB-455E-B35C-945A745520CC}.Release|Any CPU.Build.0 = Release|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 0b8535f512d235057ceb7c13c591fb7567e47b24 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 17:35:53 +0800 Subject: [PATCH 04/10] First working version --- .../pocket-tts-zero-shot/Program.cs | 85 ++++++++++++++++++- .../pocket-tts-zero-shot.csproj | 1 + dotnet-examples/pocket-tts-zero-shot/run.sh | 10 +++ 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100755 dotnet-examples/pocket-tts-zero-shot/run.sh diff --git a/dotnet-examples/pocket-tts-zero-shot/Program.cs b/dotnet-examples/pocket-tts-zero-shot/Program.cs index 83fa4f4d5f..efac41de77 100644 --- a/dotnet-examples/pocket-tts-zero-shot/Program.cs +++ b/dotnet-examples/pocket-tts-zero-shot/Program.cs @@ -1,2 +1,83 @@ -// See https://aka.ms/new-console-template for more information -Console.WriteLine("Hello, World!"); +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use a non-streaming PocketTTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using SherpaOnnx; +using System.Runtime.InteropServices; + +class PocketTtsDemo +{ + static void Main(string[] args) + { + + TestEn(); + } + + static void TestEn() + { + var config = new OfflineTtsConfig(); + config.Model.Pocket.LmFlow = "./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_flow.int8.onnx"; + config.Model.Pocket.LmMain = "./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_main.int8.onnx"; + config.Model.Pocket.Encoder = "./sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx"; + config.Model.Pocket.Decoder = "./sherpa-onnx-pocket-tts-int8-2026-01-26/decoder.int8.onnx"; + config.Model.Pocket.TextConditioner = "./sherpa-onnx-pocket-tts-int8-2026-01-26/text_conditioner.onnx"; + config.Model.Pocket.VocabJson = "./sherpa-onnx-pocket-tts-int8-2026-01-26/vocab.json"; + config.Model.Pocket.TokenScoresJson = "./sherpa-onnx-pocket-tts-int8-2026-01-26/token_scores.json"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); + + var referenceWaveFilename = "./sherpa-onnx-pocket-tts-int8-2026-01-26/test_wavs/bria.wav"; + var reader = new WaveReader(referenceWaveFilename); + + genConfig.ReferenceAudio = reader.Samples; + genConfig.ReferenceSampleRate= reader.SampleRate; + + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + var MyCallback = (IntPtr samples, int n, float progress, IntPtr arg) => + { + float[] data = new float[n]; + Marshal.Copy(samples, data, 0, n); + // You can process samples here, e.g., play them. + // See ../kitten-tts-playback for how to play them + Console.WriteLine($"Progress {progress*100}%"); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + }; + + var callback = new OfflineTtsCallbackProgressWithArg(MyCallback); + + var audio = tts.GenerateWithConfig(text, genConfig, callback); + + var outputFilename = "./generated-pocket-en.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + } +} + diff --git a/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj b/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj index a8f40655a0..c73c215f19 100644 --- a/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj +++ b/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj @@ -3,6 +3,7 @@ Exe net8.0 + true pocket_tts_zero_shot enable enable diff --git a/dotnet-examples/pocket-tts-zero-shot/run.sh b/dotnet-examples/pocket-tts-zero-shot/run.sh new file mode 100755 index 0000000000..7b23aa341f --- /dev/null +++ b/dotnet-examples/pocket-tts-zero-shot/run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 + tar xvf sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 + rm sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 +fi + +dotnet run From fbd16861c0b6ea34f454bc900dd9fae19e5397c4 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 17:36:31 +0800 Subject: [PATCH 05/10] First working version --- scripts/dotnet/OfflineTts.cs | 12 +++- scripts/dotnet/OfflineTtsGenerationConfig.cs | 62 ++++++++++++-------- scripts/dotnet/OfflineTtsModelConfig.cs | 2 + 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index e4bb248790..299e8b4880 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -49,17 +49,25 @@ public OfflineTtsGeneratedAudio GenerateWithCallbackProgress(String text, float return new OfflineTtsGeneratedAudio(p); } - public OfflineTtsGeneratedAudio GenerateWithConfig(String text, OfflineTtsGenerationConfig confi, OfflineTtsCallbackProgressWithArg callback) + public OfflineTtsGeneratedAudio GenerateWithConfig(String text, OfflineTtsGenerationConfig config, OfflineTtsCallbackProgressWithArg callback) { byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator - OfflineTtsGenerationConfig.NativeStruct nativeConfig = config.ToNative(); + GCHandle? audioHandle; + + OfflineTtsGenerationConfig.NativeStruct nativeConfig = config.ToNative(out audioHandle); IntPtr p = SherpaOnnxOfflineTtsGenerateWithConfig(_handle.Handle, utf8BytesWithNull, ref nativeConfig, callback, IntPtr.Zero); + + if (audioHandle.HasValue) + { + audioHandle.Value.Free(); + } + return new OfflineTtsGeneratedAudio(p); } diff --git a/scripts/dotnet/OfflineTtsGenerationConfig.cs b/scripts/dotnet/OfflineTtsGenerationConfig.cs index a3d54645a2..d4530c29e0 100644 --- a/scripts/dotnet/OfflineTtsGenerationConfig.cs +++ b/scripts/dotnet/OfflineTtsGenerationConfig.cs @@ -8,6 +8,18 @@ namespace SherpaOnnx { public class OfflineTtsGenerationConfig { + public OfflineTtsGenerationConfig() + { + SilenceScale = 0.2f; + Speed = 1.0f; + Sid = 0; + ReferenceAudio = null; + ReferenceSampleRate = 0; + ReferenceText = ""; + NumSteps = 5; + Extra = new Hashtable(); + } + public float SilenceScale; public float Speed; public int Sid; @@ -20,48 +32,42 @@ public class OfflineTtsGenerationConfig /// /// Extra attributes serialized as JSON manually /// - public Hashtable Extra; + public Hashtable Extra; - public OfflineTtsGenerationConfig() - { - SilenceScale = 1.0f; - Speed = 1.0f; - Sid = 0; - ReferenceAudio = null; - ReferenceSampleRate = 16000; - ReferenceText = ""; - NumSteps = 0; - Extra = new Hashtable(); - } - - internal NativeStruct ToNative() + internal NativeStruct ToNative(out GCHandle? audioHandle) { NativeStruct native = new NativeStruct(); native.SilenceScale = SilenceScale; native.Speed = Speed; native.Sid = Sid; - native.ReferenceAudio = ReferenceAudio; - native.ReferenceAudioLen = (ReferenceAudio != null) ? ReferenceAudio.Length : 0; - native.ReferenceSampleRate = ReferenceSampleRate; - native.ReferenceText = (ReferenceText != null) ? ReferenceText : ""; + audioHandle = null; + if (ReferenceAudio != null && ReferenceAudio.Length > 0) + { + audioHandle = GCHandle.Alloc(ReferenceAudio, GCHandleType.Pinned); + native.ReferenceAudio = audioHandle.Value.AddrOfPinnedObject(); + native.ReferenceAudioLen = ReferenceAudio.Length; + } + else + { + native.ReferenceAudio = IntPtr.Zero; + native.ReferenceAudioLen = 0; + } + + native.ReferenceSampleRate = ReferenceSampleRate; + native.ReferenceText = ReferenceText ?? ""; native.NumSteps = NumSteps; - // Simple JSON string generation native.Extra = "{}"; if (Extra != null && Extra.Count > 0) { string json = "{"; bool first = true; - foreach (DictionaryEntry kv in Extra) + foreach (System.Collections.DictionaryEntry kv in Extra) { if (!first) json += ","; string key = kv.Key.ToString(); - string val; - if (kv.Value is string) - val = "\"" + kv.Value.ToString() + "\""; - else - val = kv.Value.ToString(); + string val = kv.Value is string ? "\"" + kv.Value.ToString() + "\"" : kv.Value.ToString(); json += "\"" + key + "\":" + val; first = false; } @@ -72,6 +78,9 @@ internal NativeStruct ToNative() return native; } + + + [StructLayout(LayoutKind.Sequential)] internal struct NativeStruct { @@ -79,7 +88,7 @@ internal struct NativeStruct public float Speed; public int Sid; - public float[] ReferenceAudio; + public IntPtr ReferenceAudio; // Use IntPtr for dynamic array public int ReferenceAudioLen; public int ReferenceSampleRate; @@ -91,6 +100,7 @@ internal struct NativeStruct [MarshalAs(UnmanagedType.LPStr)] public string Extra; } + } } diff --git a/scripts/dotnet/OfflineTtsModelConfig.cs b/scripts/dotnet/OfflineTtsModelConfig.cs index b4d72ec5f8..e1c2cacc61 100644 --- a/scripts/dotnet/OfflineTtsModelConfig.cs +++ b/scripts/dotnet/OfflineTtsModelConfig.cs @@ -14,6 +14,7 @@ public OfflineTtsModelConfig() Kokoro = new OfflineTtsKokoroModelConfig(); Kitten = new OfflineTtsKittenModelConfig(); ZipVoice = new OfflineTtsZipVoiceModelConfig(); + Pocket = new OfflineTtsPocketModelConfig(); NumThreads = 1; Debug = 0; Provider = "cpu"; @@ -30,5 +31,6 @@ public OfflineTtsModelConfig() public OfflineTtsKokoroModelConfig Kokoro; public OfflineTtsKittenModelConfig Kitten; public OfflineTtsZipVoiceModelConfig ZipVoice; + public OfflineTtsPocketModelConfig Pocket; } } From b7ccda87a49729d7d41a940727282880447ff836 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 17:44:01 +0800 Subject: [PATCH 06/10] Add C# API for PocketTTS. --- .github/scripts/test-dot-net.sh | 7 +- .../pocket-tts-zero-shot-play/Program.cs | 206 ++++++++++++++++++ .../pocket-tts-zero-shot-play.csproj | 19 ++ .../pocket-tts-zero-shot-play/run.sh | 10 + .../pocket-tts-zero-shot.csproj | 5 +- dotnet-examples/sherpa-onnx.sln | 6 + 6 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 dotnet-examples/pocket-tts-zero-shot-play/Program.cs create mode 100644 dotnet-examples/pocket-tts-zero-shot-play/pocket-tts-zero-shot-play.csproj create mode 100755 dotnet-examples/pocket-tts-zero-shot-play/run.sh diff --git a/.github/scripts/test-dot-net.sh b/.github/scripts/test-dot-net.sh index ffd5cf1bee..3ffc5f2081 100755 --- a/.github/scripts/test-dot-net.sh +++ b/.github/scripts/test-dot-net.sh @@ -4,7 +4,12 @@ set -ex cd dotnet-examples/ -cd ./vad-non-streaming-funasr-nano +cd ./pocket-tts-zero-shot +./run.sh +ls -lh +rm -rf sherpa-onnx-pocket-* + +cd ../vad-non-streaming-funasr-nano ./run-ten-vad.sh rm -fv *.onnx diff --git a/dotnet-examples/pocket-tts-zero-shot-play/Program.cs b/dotnet-examples/pocket-tts-zero-shot-play/Program.cs new file mode 100644 index 0000000000..235ec91eca --- /dev/null +++ b/dotnet-examples/pocket-tts-zero-shot-play/Program.cs @@ -0,0 +1,206 @@ +// Copyright (c) 2026 Xiaomi Corporation +// +// This file shows how to use a non-streaming PocketTTS model +// for text-to-speech +// Please refer to +// https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html +// and +// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models +// to download pre-trained models +using PortAudioSharp; +using SherpaOnnx; +using System.Collections.Concurrent; +using System.Runtime.InteropServices; + +class PocketTtsDemo +{ + static void Main(string[] args) + { + + TestEn(); + } + + static void TestEn() + { + var config = new OfflineTtsConfig(); + config.Model.Pocket.LmFlow = "./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_flow.int8.onnx"; + config.Model.Pocket.LmMain = "./sherpa-onnx-pocket-tts-int8-2026-01-26/lm_main.int8.onnx"; + config.Model.Pocket.Encoder = "./sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx"; + config.Model.Pocket.Decoder = "./sherpa-onnx-pocket-tts-int8-2026-01-26/decoder.int8.onnx"; + config.Model.Pocket.TextConditioner = "./sherpa-onnx-pocket-tts-int8-2026-01-26/text_conditioner.onnx"; + config.Model.Pocket.VocabJson = "./sherpa-onnx-pocket-tts-int8-2026-01-26/vocab.json"; + config.Model.Pocket.TokenScoresJson = "./sherpa-onnx-pocket-tts-int8-2026-01-26/token_scores.json"; + + config.Model.NumThreads = 2; + config.Model.Debug = 1; + config.Model.Provider = "cpu"; + + OfflineTtsGenerationConfig genConfig = new OfflineTtsGenerationConfig(); + + var referenceWaveFilename = "./sherpa-onnx-pocket-tts-int8-2026-01-26/test_wavs/bria.wav"; + var reader = new WaveReader(referenceWaveFilename); + + genConfig.ReferenceAudio = reader.Samples; + genConfig.ReferenceSampleRate= reader.SampleRate; + + + var tts = new OfflineTts(config); + var speed = 1.0f; + var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + + "does not have two-thirds of his day for himself, is a slave, whatever " + + "he may be: a statesman, a businessman, an official, or a scholar. " + + "Friends fell out often because life was changing so fast. The easiest " + + "thing in the world was to lose touch with someone."; + + Console.WriteLine(PortAudio.VersionInfo.versionText); + PortAudio.Initialize(); + Console.WriteLine($"Number of devices: {PortAudio.DeviceCount}"); + + for (int i = 0; i != PortAudio.DeviceCount; ++i) + { + Console.WriteLine($" Device {i}"); + DeviceInfo deviceInfo = PortAudio.GetDeviceInfo(i); + Console.WriteLine($" Name: {deviceInfo.name}"); + Console.WriteLine($" Max output channels: {deviceInfo.maxOutputChannels}"); + Console.WriteLine($" Default sample rate: {deviceInfo.defaultSampleRate}"); + } + int deviceIndex = PortAudio.DefaultOutputDevice; + if (deviceIndex == PortAudio.NoDevice) + { + Console.WriteLine("No default output device found. Please use ../offline-tts instead"); + Environment.Exit(1); + } + + var info = PortAudio.GetDeviceInfo(deviceIndex); + Console.WriteLine(); + Console.WriteLine($"Use output default device {deviceIndex} ({info.name})"); + + var param = new StreamParameters(); + param.device = deviceIndex; + param.channelCount = 1; + param.sampleFormat = SampleFormat.Float32; + param.suggestedLatency = info.defaultLowOutputLatency; + param.hostApiSpecificStreamInfo = IntPtr.Zero; + + // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview + var dataItems = new BlockingCollection(); + + var MyCallback = (IntPtr samples, int n, float progress, IntPtr arg) => + { + Console.WriteLine($"Progress {progress*100}%"); + + float[] data = new float[n]; + + Marshal.Copy(samples, data, 0, n); + + dataItems.Add(data); + + // 1 means to keep generating + // 0 means to stop generating + return 1; + + }; + + + var playFinished = false; + + float[]? lastSampleArray = null; + int lastIndex = 0; // not played + + PortAudioSharp.Stream.Callback playCallback = (IntPtr input, IntPtr output, + UInt32 frameCount, + ref StreamCallbackTimeInfo timeInfo, + StreamCallbackFlags statusFlags, + IntPtr userData + ) => + { + if (dataItems.IsCompleted && lastSampleArray == null && lastIndex == 0) + { + Console.WriteLine($"Finished playing"); + playFinished = true; + return StreamCallbackResult.Complete; + } + + int expected = Convert.ToInt32(frameCount); + int i = 0; + + while ((lastSampleArray != null || dataItems.Count != 0) && (i < expected)) + { + int needed = expected - i; + + if (lastSampleArray != null) + { + int remaining = lastSampleArray.Length - lastIndex; + if (remaining >= needed) + { + float[] this_block = lastSampleArray.Skip(lastIndex).Take(needed).ToArray(); + lastIndex += needed; + if (lastIndex == lastSampleArray.Length) + { + lastSampleArray = null; + lastIndex = 0; + } + + Marshal.Copy(this_block, 0, IntPtr.Add(output, i * sizeof(float)), needed); + return StreamCallbackResult.Continue; + } + + float[] this_block2 = lastSampleArray.Skip(lastIndex).Take(remaining).ToArray(); + lastIndex = 0; + lastSampleArray = null; + + Marshal.Copy(this_block2, 0, IntPtr.Add(output, i * sizeof(float)), remaining); + i += remaining; + continue; + } + + if (dataItems.Count != 0) + { + lastSampleArray = dataItems.Take(); + lastIndex = 0; + } + } + + if (i < expected) + { + int sizeInBytes = (expected - i) * 4; + Marshal.Copy(new byte[sizeInBytes], 0, IntPtr.Add(output, i * sizeof(float)), sizeInBytes); + } + + return StreamCallbackResult.Continue; + }; + + PortAudioSharp.Stream stream = new PortAudioSharp.Stream(inParams: null, outParams: param, sampleRate: tts.SampleRate, + framesPerBuffer: 0, + streamFlags: StreamFlags.ClipOff, + callback: playCallback, + userData: IntPtr.Zero + ); + + stream.Start(); + + var callback = new OfflineTtsCallbackProgressWithArg(MyCallback); + + var audio = tts.GenerateWithConfig(text, genConfig, callback); + + var outputFilename = "./generated-pocket-en-paly.wav"; + var ok = audio.SaveToWaveFile(outputFilename); + + if (ok) + { + Console.WriteLine($"Wrote to {outputFilename} succeeded!"); + } + else + { + Console.WriteLine($"Failed to write {outputFilename}"); + } + + dataItems.CompleteAdding(); + + while (!playFinished) + { + Thread.Sleep(100); // 100ms + } + } +} + diff --git a/dotnet-examples/pocket-tts-zero-shot-play/pocket-tts-zero-shot-play.csproj b/dotnet-examples/pocket-tts-zero-shot-play/pocket-tts-zero-shot-play.csproj new file mode 100644 index 0000000000..beb0843b50 --- /dev/null +++ b/dotnet-examples/pocket-tts-zero-shot-play/pocket-tts-zero-shot-play.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + pocket_tts_zero_shot_play + enable + enable + + + + + + + + + + + diff --git a/dotnet-examples/pocket-tts-zero-shot-play/run.sh b/dotnet-examples/pocket-tts-zero-shot-play/run.sh new file mode 100755 index 0000000000..7b23aa341f --- /dev/null +++ b/dotnet-examples/pocket-tts-zero-shot-play/run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +if [ ! -f ./sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 + tar xvf sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 + rm sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2 +fi + +dotnet run diff --git a/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj b/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj index c73c215f19..09e8b4ea2f 100644 --- a/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj +++ b/dotnet-examples/pocket-tts-zero-shot/pocket-tts-zero-shot.csproj @@ -3,10 +3,13 @@ Exe net8.0 - true pocket_tts_zero_shot enable enable + + + + diff --git a/dotnet-examples/sherpa-onnx.sln b/dotnet-examples/sherpa-onnx.sln index 0c0a7a3294..2bfc748ad6 100644 --- a/dotnet-examples/sherpa-onnx.sln +++ b/dotnet-examples/sherpa-onnx.sln @@ -53,6 +53,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "vad-non-streaming-funasr-na EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "pocket-tts-zero-shot", "pocket-tts-zero-shot\pocket-tts-zero-shot.csproj", "{9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "pocket-tts-zero-shot-play", "pocket-tts-zero-shot-play\pocket-tts-zero-shot-play.csproj", "{0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -159,6 +161,10 @@ Global {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Debug|Any CPU.Build.0 = Debug|Any CPU {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.ActiveCfg = Release|Any CPU {9164FA6A-F8D3-4F52-8173-A2FA78E74BB2}.Release|Any CPU.Build.0 = Release|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0E73BD08-EA6F-416D-8DBF-E92893A8C3B1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 2b011d9e5525c33670b60e98dc44f0dc2c3c9a80 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 18:08:09 +0800 Subject: [PATCH 07/10] Fix typos --- dotnet-examples/pocket-tts-zero-shot-play/Program.cs | 3 +-- dotnet-examples/pocket-tts-zero-shot/Program.cs | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dotnet-examples/pocket-tts-zero-shot-play/Program.cs b/dotnet-examples/pocket-tts-zero-shot-play/Program.cs index 235ec91eca..8b408d05a2 100644 --- a/dotnet-examples/pocket-tts-zero-shot-play/Program.cs +++ b/dotnet-examples/pocket-tts-zero-shot-play/Program.cs @@ -42,10 +42,9 @@ static void TestEn() genConfig.ReferenceAudio = reader.Samples; genConfig.ReferenceSampleRate= reader.SampleRate; - + genConfig.Extra["max_reference_audio_len"] = 12; var tts = new OfflineTts(config); - var speed = 1.0f; var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + "does not have two-thirds of his day for himself, is a slave, whatever " + "he may be: a statesman, a businessman, an official, or a scholar. " + diff --git a/dotnet-examples/pocket-tts-zero-shot/Program.cs b/dotnet-examples/pocket-tts-zero-shot/Program.cs index efac41de77..8c03f273e4 100644 --- a/dotnet-examples/pocket-tts-zero-shot/Program.cs +++ b/dotnet-examples/pocket-tts-zero-shot/Program.cs @@ -39,11 +39,10 @@ static void TestEn() var reader = new WaveReader(referenceWaveFilename); genConfig.ReferenceAudio = reader.Samples; - genConfig.ReferenceSampleRate= reader.SampleRate; - + genConfig.ReferenceSampleRate = reader.SampleRate; + genConfig.Extra["max_reference_audio_len"] = 12; var tts = new OfflineTts(config); - var speed = 1.0f; var text = "Today as always, men fall into two groups: slaves and free men. Whoever " + "does not have two-thirds of his day for himself, is a slave, whatever " + "he may be: a statesman, a businessman, an official, or a scholar. " + From 23c5785a78f1b4df8ab00442285e2d25de15e863 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 19:41:33 +0800 Subject: [PATCH 08/10] Fixes after review --- dotnet-examples/kitten-tts-play/Program.cs | 4 +- dotnet-examples/kitten-tts/Program.cs | 2 +- dotnet-examples/offline-tts-play/Program.cs | 4 +- .../pocket-tts-zero-shot-play/Program.cs | 6 +- .../pocket-tts-zero-shot/Program.cs | 2 +- scripts/dotnet/OfflineTts.cs | 47 +++++++--- scripts/dotnet/OfflineTtsGenerationConfig.cs | 88 ++++++++++++++++--- 7 files changed, 118 insertions(+), 35 deletions(-) diff --git a/dotnet-examples/kitten-tts-play/Program.cs b/dotnet-examples/kitten-tts-play/Program.cs index 62ab00657a..8e01e0216e 100644 --- a/dotnet-examples/kitten-tts-play/Program.cs +++ b/dotnet-examples/kitten-tts-play/Program.cs @@ -74,7 +74,7 @@ static void Main(string[] args) // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview var dataItems = new BlockingCollection(); - var MyCallback = (IntPtr samples, int n, float progress) => + var myCallback = (IntPtr samples, int n, float progress) => { Console.WriteLine($"Progress {progress*100}%"); @@ -166,7 +166,7 @@ IntPtr userData stream.Start(); - var callback = new OfflineTtsCallbackProgress(MyCallback); + var callback = new OfflineTtsCallbackProgress(myCallback); var audio = tts.GenerateWithCallbackProgress(text, speed, sid, callback); var outputFilename = "./generated-kitten-0.wav"; diff --git a/dotnet-examples/kitten-tts/Program.cs b/dotnet-examples/kitten-tts/Program.cs index 6052f3124a..b45fae4319 100644 --- a/dotnet-examples/kitten-tts/Program.cs +++ b/dotnet-examples/kitten-tts/Program.cs @@ -49,7 +49,7 @@ static void TestEn() float[] data = new float[n]; Marshal.Copy(samples, data, 0, n); // You can process samples here, e.g., play them. - // See ../kitten-tts-playback for how to play them + // See ../kitten-tts-play for how to play them Console.WriteLine($"Progress {progress*100}%"); // 1 means to keep generating diff --git a/dotnet-examples/offline-tts-play/Program.cs b/dotnet-examples/offline-tts-play/Program.cs index 76866337d4..24aadfebfc 100644 --- a/dotnet-examples/offline-tts-play/Program.cs +++ b/dotnet-examples/offline-tts-play/Program.cs @@ -224,7 +224,7 @@ private static void Run(Options options) // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview var dataItems = new BlockingCollection(); - var MyCallback = (IntPtr samples, int n) => + var myCallback = (IntPtr samples, int n) => { float[] data = new float[n]; @@ -314,7 +314,7 @@ IntPtr userData stream.Start(); - var callback = new OfflineTtsCallback(MyCallback); + var callback = new OfflineTtsCallback(myCallback); var audio = tts.GenerateWithCallback(options.Text, speed, sid, callback); var ok = audio.SaveToWaveFile(options.OutputFilename); diff --git a/dotnet-examples/pocket-tts-zero-shot-play/Program.cs b/dotnet-examples/pocket-tts-zero-shot-play/Program.cs index 8b408d05a2..37ce23f652 100644 --- a/dotnet-examples/pocket-tts-zero-shot-play/Program.cs +++ b/dotnet-examples/pocket-tts-zero-shot-play/Program.cs @@ -84,7 +84,7 @@ static void TestEn() // https://learn.microsoft.com/en-us/dotnet/standard/collections/thread-safe/blockingcollection-overview var dataItems = new BlockingCollection(); - var MyCallback = (IntPtr samples, int n, float progress, IntPtr arg) => + var myCallback = (IntPtr samples, int n, float progress, IntPtr arg) => { Console.WriteLine($"Progress {progress*100}%"); @@ -178,11 +178,11 @@ IntPtr userData stream.Start(); - var callback = new OfflineTtsCallbackProgressWithArg(MyCallback); + var callback = new OfflineTtsCallbackProgressWithArg(myCallback); var audio = tts.GenerateWithConfig(text, genConfig, callback); - var outputFilename = "./generated-pocket-en-paly.wav"; + var outputFilename = "./generated-pocket-en-play.wav"; var ok = audio.SaveToWaveFile(outputFilename); if (ok) diff --git a/dotnet-examples/pocket-tts-zero-shot/Program.cs b/dotnet-examples/pocket-tts-zero-shot/Program.cs index 8c03f273e4..8daac85092 100644 --- a/dotnet-examples/pocket-tts-zero-shot/Program.cs +++ b/dotnet-examples/pocket-tts-zero-shot/Program.cs @@ -54,7 +54,7 @@ static void TestEn() float[] data = new float[n]; Marshal.Copy(samples, data, 0, n); // You can process samples here, e.g., play them. - // See ../kitten-tts-playback for how to play them + // See ../pocket-tts-zero-shot-play for how to play them Console.WriteLine($"Progress {progress*100}%"); // 1 means to keep generating diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index 299e8b4880..d30f30bb5d 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -6,13 +6,24 @@ namespace SherpaOnnx { // IntPtr is actually a `const float*` from C++ + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate int OfflineTtsCallback(IntPtr samples, int n); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate int OfflineTtsCallbackProgress(IntPtr samples, int n, float progress); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate int OfflineTtsCallbackProgressWithArg(IntPtr samples, int n, float progress, IntPtr arg); public class OfflineTts : IDisposable { + // Keep delegates alive + private OfflineTtsCallback _callbackRef; + private OfflineTtsCallbackProgress _callbackProgressRef; + private OfflineTtsCallbackProgressWithArg _callbackWithArgRef; + + public OfflineTts(OfflineTtsConfig config) { IntPtr h = SherpaOnnxCreateOfflineTts(ref config); @@ -31,46 +42,54 @@ public OfflineTtsGeneratedAudio Generate(String text, float speed, int speakerId public OfflineTtsGeneratedAudio GenerateWithCallback(String text, float speed, int speakerId, OfflineTtsCallback callback) { + _callbackRef = callback; byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator - IntPtr p = SherpaOnnxOfflineTtsGenerateWithCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, callback); + IntPtr p = SherpaOnnxOfflineTtsGenerateWithCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, _callbackRef); + _callbackRef = null; return new OfflineTtsGeneratedAudio(p); } public OfflineTtsGeneratedAudio GenerateWithCallbackProgress(String text, float speed, int speakerId, OfflineTtsCallbackProgress callback) { + _callbackProgressRef = callback; byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator - IntPtr p = SherpaOnnxOfflineTtsGenerateWithProgressCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, callback); + IntPtr p = SherpaOnnxOfflineTtsGenerateWithProgressCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, _callbackProgressRef); + _callbackProgressRef = null; return new OfflineTtsGeneratedAudio(p); } - public OfflineTtsGeneratedAudio GenerateWithConfig(String text, OfflineTtsGenerationConfig config, OfflineTtsCallbackProgressWithArg callback) + public OfflineTtsGeneratedAudio GenerateWithConfig(string text, OfflineTtsGenerationConfig config, OfflineTtsCallbackProgressWithArg callback) { - byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); - byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator - Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); - utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator - - GCHandle? audioHandle; - - OfflineTtsGenerationConfig.NativeStruct nativeConfig = config.ToNative(out audioHandle); + _callbackWithArgRef = callback; + byte[] utf8BytesWithNull = GetUtf8BytesWithNull(text); + GCHandle? audioHandle = null; + IntPtr p; - IntPtr p = SherpaOnnxOfflineTtsGenerateWithConfig(_handle.Handle, utf8BytesWithNull, ref nativeConfig, callback, IntPtr.Zero); + var nativeConfig = config.ToNative(out audioHandle); - if (audioHandle.HasValue) + try { - audioHandle.Value.Free(); + p = SherpaOnnxOfflineTtsGenerateWithConfig(_handle.Handle, utf8BytesWithNull, ref nativeConfig, _callbackWithArgRef, IntPtr.Zero); + } + finally + { + // Ensure the pinned handle is freed even if P/Invoke throws + if (audioHandle.HasValue) + audioHandle.Value.Free(); } + _callbackWithArgRef = null; // safe: blocking call return new OfflineTtsGeneratedAudio(p); } + public void Dispose() { Cleanup(); diff --git a/scripts/dotnet/OfflineTtsGenerationConfig.cs b/scripts/dotnet/OfflineTtsGenerationConfig.cs index d4530c29e0..40386bc543 100644 --- a/scripts/dotnet/OfflineTtsGenerationConfig.cs +++ b/scripts/dotnet/OfflineTtsGenerationConfig.cs @@ -3,6 +3,11 @@ using System; using System.Collections; using System.Runtime.InteropServices; +using System.Text; + +#if !NET20 +using System.Web.Script.Serialization; +#endif namespace SherpaOnnx { @@ -30,10 +35,13 @@ public OfflineTtsGenerationConfig() public int NumSteps; /// - /// Extra attributes serialized as JSON manually + /// Extra attributes stored as key/value pairs /// public Hashtable Extra; + /// + /// Convert to native struct for P/Invoke + /// internal NativeStruct ToNative(out GCHandle? audioHandle) { NativeStruct native = new NativeStruct(); @@ -41,6 +49,7 @@ internal NativeStruct ToNative(out GCHandle? audioHandle) native.Speed = Speed; native.Sid = Sid; + // Handle ReferenceAudio audioHandle = null; if (ReferenceAudio != null && ReferenceAudio.Length > 0) { @@ -58,28 +67,84 @@ internal NativeStruct ToNative(out GCHandle? audioHandle) native.ReferenceText = ReferenceText ?? ""; native.NumSteps = NumSteps; + // Handle Extra JSON +#if NET20 native.Extra = "{}"; if (Extra != null && Extra.Count > 0) { - string json = "{"; + StringBuilder json = new StringBuilder(); + json.Append("{"); bool first = true; - foreach (System.Collections.DictionaryEntry kv in Extra) + + foreach (DictionaryEntry kv in Extra) { - if (!first) json += ","; - string key = kv.Key.ToString(); - string val = kv.Value is string ? "\"" + kv.Value.ToString() + "\"" : kv.Value.ToString(); - json += "\"" + key + "\":" + val; + if (!first) json.Append(","); first = false; + + string key = JsonEscape(kv.Key.ToString()); + string val; + + if (kv.Value is string) + val = JsonEscape((string)kv.Value); + else if (kv.Value is float || kv.Value is double) + val = ((IFormattable)kv.Value).ToString(null, System.Globalization.CultureInfo.InvariantCulture); + else + val = kv.Value.ToString(); + + json.AppendFormat("{0}:{1}", key, val); } - json += "}"; - native.Extra = json; + + json.Append("}"); + native.Extra = json.ToString(); + } +#else + if (Extra != null && Extra.Count > 0) + { + var serializer = new JavaScriptSerializer(); + native.Extra = serializer.Serialize(Extra); + } + else + { + native.Extra = "{}"; } +#endif return native; } +#if NET20 + /// + /// Escapes a string for JSON (for .NET 2.0) + /// + private static string JsonEscape(string s) + { + if (s == null) return "\"\""; - + StringBuilder sb = new StringBuilder(); + sb.Append('"'); + foreach (char c in s) + { + switch (c) + { + case '"': sb.Append("\\\""); break; + case '\\': sb.Append("\\\\"); break; + case '\b': sb.Append("\\b"); break; + case '\f': sb.Append("\\f"); break; + case '\n': sb.Append("\\n"); break; + case '\r': sb.Append("\\r"); break; + case '\t': sb.Append("\\t"); break; + default: + if (c < 32 || c > 126) + sb.AppendFormat("\\u{0:X4}", (int)c); + else + sb.Append(c); + break; + } + } + sb.Append('"'); + return sb.ToString(); + } +#endif [StructLayout(LayoutKind.Sequential)] internal struct NativeStruct @@ -88,7 +153,7 @@ internal struct NativeStruct public float Speed; public int Sid; - public IntPtr ReferenceAudio; // Use IntPtr for dynamic array + public IntPtr ReferenceAudio; public int ReferenceAudioLen; public int ReferenceSampleRate; @@ -100,7 +165,6 @@ internal struct NativeStruct [MarshalAs(UnmanagedType.LPStr)] public string Extra; } - } } From 4b0b0e4d3a323129e44215449b98128132376005 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 9 Feb 2026 21:15:14 +0800 Subject: [PATCH 09/10] Fixes after review --- scripts/dotnet/OfflineTts.cs | 4 +- scripts/dotnet/OfflineTtsGenerationConfig.cs | 346 ++++++++++--------- 2 files changed, 179 insertions(+), 171 deletions(-) diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index d30f30bb5d..89333460df 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -67,7 +67,9 @@ public OfflineTtsGeneratedAudio GenerateWithCallbackProgress(String text, float public OfflineTtsGeneratedAudio GenerateWithConfig(string text, OfflineTtsGenerationConfig config, OfflineTtsCallbackProgressWithArg callback) { _callbackWithArgRef = callback; - byte[] utf8BytesWithNull = GetUtf8BytesWithNull(text); + byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); GCHandle? audioHandle = null; IntPtr p; diff --git a/scripts/dotnet/OfflineTtsGenerationConfig.cs b/scripts/dotnet/OfflineTtsGenerationConfig.cs index 40386bc543..390ac25db3 100644 --- a/scripts/dotnet/OfflineTtsGenerationConfig.cs +++ b/scripts/dotnet/OfflineTtsGenerationConfig.cs @@ -1,170 +1,176 @@ -/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) - -using System; -using System.Collections; -using System.Runtime.InteropServices; -using System.Text; - -#if !NET20 -using System.Web.Script.Serialization; -#endif - -namespace SherpaOnnx -{ - public class OfflineTtsGenerationConfig - { - public OfflineTtsGenerationConfig() - { - SilenceScale = 0.2f; - Speed = 1.0f; - Sid = 0; - ReferenceAudio = null; - ReferenceSampleRate = 0; - ReferenceText = ""; - NumSteps = 5; - Extra = new Hashtable(); - } - - public float SilenceScale; - public float Speed; - public int Sid; - - public float[] ReferenceAudio; - public int ReferenceSampleRate; - public string ReferenceText; - public int NumSteps; - - /// - /// Extra attributes stored as key/value pairs - /// - public Hashtable Extra; - - /// - /// Convert to native struct for P/Invoke - /// - internal NativeStruct ToNative(out GCHandle? audioHandle) - { - NativeStruct native = new NativeStruct(); - native.SilenceScale = SilenceScale; - native.Speed = Speed; - native.Sid = Sid; - - // Handle ReferenceAudio - audioHandle = null; - if (ReferenceAudio != null && ReferenceAudio.Length > 0) - { - audioHandle = GCHandle.Alloc(ReferenceAudio, GCHandleType.Pinned); - native.ReferenceAudio = audioHandle.Value.AddrOfPinnedObject(); - native.ReferenceAudioLen = ReferenceAudio.Length; - } - else - { - native.ReferenceAudio = IntPtr.Zero; - native.ReferenceAudioLen = 0; - } - - native.ReferenceSampleRate = ReferenceSampleRate; - native.ReferenceText = ReferenceText ?? ""; - native.NumSteps = NumSteps; - - // Handle Extra JSON -#if NET20 - native.Extra = "{}"; - if (Extra != null && Extra.Count > 0) - { - StringBuilder json = new StringBuilder(); - json.Append("{"); - bool first = true; - - foreach (DictionaryEntry kv in Extra) - { - if (!first) json.Append(","); - first = false; - - string key = JsonEscape(kv.Key.ToString()); - string val; - - if (kv.Value is string) - val = JsonEscape((string)kv.Value); - else if (kv.Value is float || kv.Value is double) - val = ((IFormattable)kv.Value).ToString(null, System.Globalization.CultureInfo.InvariantCulture); - else - val = kv.Value.ToString(); - - json.AppendFormat("{0}:{1}", key, val); - } - - json.Append("}"); - native.Extra = json.ToString(); - } -#else - if (Extra != null && Extra.Count > 0) - { - var serializer = new JavaScriptSerializer(); - native.Extra = serializer.Serialize(Extra); - } - else - { - native.Extra = "{}"; - } -#endif - - return native; - } - -#if NET20 - /// - /// Escapes a string for JSON (for .NET 2.0) - /// - private static string JsonEscape(string s) - { - if (s == null) return "\"\""; - - StringBuilder sb = new StringBuilder(); - sb.Append('"'); - foreach (char c in s) - { - switch (c) - { - case '"': sb.Append("\\\""); break; - case '\\': sb.Append("\\\\"); break; - case '\b': sb.Append("\\b"); break; - case '\f': sb.Append("\\f"); break; - case '\n': sb.Append("\\n"); break; - case '\r': sb.Append("\\r"); break; - case '\t': sb.Append("\\t"); break; - default: - if (c < 32 || c > 126) - sb.AppendFormat("\\u{0:X4}", (int)c); - else - sb.Append(c); - break; - } - } - sb.Append('"'); - return sb.ToString(); - } -#endif - - [StructLayout(LayoutKind.Sequential)] - internal struct NativeStruct - { - public float SilenceScale; - public float Speed; - public int Sid; - - public IntPtr ReferenceAudio; - public int ReferenceAudioLen; - public int ReferenceSampleRate; - - [MarshalAs(UnmanagedType.LPStr)] - public string ReferenceText; - - public int NumSteps; - - [MarshalAs(UnmanagedType.LPStr)] - public string Extra; - } - } -} - +/// Copyright (c) 2026 Xiaomi Corporation (authors: Fangjun Kuang) + +using System; +using System.Collections; +using System.Runtime.InteropServices; +using System.Text; + +#if !NET20 +using System.Text.Json; +#endif + +namespace SherpaOnnx +{ + public class OfflineTtsGenerationConfig + { + public OfflineTtsGenerationConfig() + { + SilenceScale = 0.2f; + Speed = 1.0f; + Sid = 0; + ReferenceAudio = null; + ReferenceSampleRate = 0; + ReferenceText = ""; + NumSteps = 5; + Extra = new Hashtable(); + } + + public float SilenceScale; + public float Speed; + public int Sid; + + public float[] ReferenceAudio; + public int ReferenceSampleRate; + public string ReferenceText; + public int NumSteps; + + /// + /// Extra attributes stored as key/value pairs + /// + public Hashtable Extra; + + /// + /// Convert to native struct for P/Invoke + /// + internal NativeStruct ToNative(out GCHandle? audioHandle) + { + NativeStruct native = new NativeStruct(); + native.SilenceScale = SilenceScale; + native.Speed = Speed; + native.Sid = Sid; + + // Handle ReferenceAudio + audioHandle = null; + if (ReferenceAudio != null && ReferenceAudio.Length > 0) + { + audioHandle = GCHandle.Alloc(ReferenceAudio, GCHandleType.Pinned); + native.ReferenceAudio = audioHandle.Value.AddrOfPinnedObject(); + native.ReferenceAudioLen = ReferenceAudio.Length; + } + else + { + native.ReferenceAudio = IntPtr.Zero; + native.ReferenceAudioLen = 0; + } + + native.ReferenceSampleRate = ReferenceSampleRate; + native.ReferenceText = ReferenceText ?? ""; + native.NumSteps = NumSteps; + + // Handle Extra JSON +#if NET20 + native.Extra = "{}"; + if (Extra != null && Extra.Count > 0) + { + StringBuilder json = new StringBuilder(); + json.Append("{"); + bool first = true; + + foreach (DictionaryEntry kv in Extra) + { + if (!first) json.Append(","); + first = false; + + string key = JsonEscape(kv.Key.ToString()); + string val; + + if (kv.Value is string) + val = JsonEscape((string)kv.Value); + else if (kv.Value is float || kv.Value is double) + val = ((IFormattable)kv.Value).ToString(null, System.Globalization.CultureInfo.InvariantCulture); + else if (kv.Value is bool) + val = (bool)kv.Value ? "true" : "false"; + else + val = kv.Value.ToString(); + + json.AppendFormat("{0}:{1}", key, val); + } + + json.Append("}"); + native.Extra = json.ToString(); + } +#else + if (Extra != null && Extra.Count > 0) + { + native.Extra = JsonSerializer.Serialize( + Extra, + new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + } + else + { + native.Extra = "{}"; + } +#endif + + return native; + } + +#if NET20 + /// + /// Escapes a string for JSON (for .NET 2.0) + /// + private static string JsonEscape(string s) + { + if (s == null) return "\"\""; + + StringBuilder sb = new StringBuilder(); + sb.Append('"'); + foreach (char c in s) + { + switch (c) + { + case '"': sb.Append("\\\""); break; + case '\\': sb.Append("\\\\"); break; + case '\b': sb.Append("\\b"); break; + case '\f': sb.Append("\\f"); break; + case '\n': sb.Append("\\n"); break; + case '\r': sb.Append("\\r"); break; + case '\t': sb.Append("\\t"); break; + default: + if (c < 32 || c > 126) + sb.AppendFormat("\\u{0:X4}", (int)c); + else + sb.Append(c); + break; + } + } + sb.Append('"'); + return sb.ToString(); + } +#endif + + [StructLayout(LayoutKind.Sequential)] + internal struct NativeStruct + { + public float SilenceScale; + public float Speed; + public int Sid; + + public IntPtr ReferenceAudio; + public int ReferenceAudioLen; + public int ReferenceSampleRate; + + [MarshalAs(UnmanagedType.LPStr)] + public string ReferenceText; + + public int NumSteps; + + [MarshalAs(UnmanagedType.LPStr)] + public string Extra; + } + } +} + From 2ee19b84099794e154d9dfb215ee3ad61759c870 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 10 Feb 2026 10:04:47 +0800 Subject: [PATCH 10/10] Fixes after review --- scripts/dotnet/OfflineTts.cs | 108 +++++++++++++------ scripts/dotnet/OfflineTtsGenerationConfig.cs | 21 ++-- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/scripts/dotnet/OfflineTts.cs b/scripts/dotnet/OfflineTts.cs index 89333460df..5fd9dcfa26 100644 --- a/scripts/dotnet/OfflineTts.cs +++ b/scripts/dotnet/OfflineTts.cs @@ -18,12 +18,6 @@ namespace SherpaOnnx public class OfflineTts : IDisposable { - // Keep delegates alive - private OfflineTtsCallback _callbackRef; - private OfflineTtsCallbackProgress _callbackProgressRef; - private OfflineTtsCallbackProgressWithArg _callbackWithArgRef; - - public OfflineTts(OfflineTtsConfig config) { IntPtr h = SherpaOnnxCreateOfflineTts(ref config); @@ -40,58 +34,112 @@ public OfflineTtsGeneratedAudio Generate(String text, float speed, int speakerId return new OfflineTtsGeneratedAudio(p); } - public OfflineTtsGeneratedAudio GenerateWithCallback(String text, float speed, int speakerId, OfflineTtsCallback callback) + public OfflineTtsGeneratedAudio GenerateWithCallback( + String text, + float speed, + int speakerId, + OfflineTtsCallback callback) { - _callbackRef = callback; byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); - byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); - utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator - IntPtr p = SherpaOnnxOfflineTtsGenerateWithCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, _callbackRef); - _callbackRef = null; - return new OfflineTtsGeneratedAudio(p); + utf8BytesWithNull[utf8Bytes.Length] = 0; + + GCHandle callbackHandle = default(GCHandle); + try + { + callbackHandle = GCHandle.Alloc(callback); + + IntPtr p = SherpaOnnxOfflineTtsGenerateWithCallback( + _handle.Handle, + utf8BytesWithNull, + speakerId, + speed, + callback + ); + + return new OfflineTtsGeneratedAudio(p); + } + finally + { + if (callbackHandle.IsAllocated) + callbackHandle.Free(); + } } - public OfflineTtsGeneratedAudio GenerateWithCallbackProgress(String text, float speed, int speakerId, OfflineTtsCallbackProgress callback) + public OfflineTtsGeneratedAudio GenerateWithCallbackProgress( + String text, + float speed, + int speakerId, + OfflineTtsCallbackProgress callback) { - _callbackProgressRef = callback; byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); - byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); - utf8BytesWithNull[utf8Bytes.Length] = 0; // Null terminator - IntPtr p = SherpaOnnxOfflineTtsGenerateWithProgressCallback(_handle.Handle, utf8BytesWithNull, speakerId, speed, _callbackProgressRef); - _callbackProgressRef = null; - return new OfflineTtsGeneratedAudio(p); + utf8BytesWithNull[utf8Bytes.Length] = 0; + + GCHandle callbackHandle = default(GCHandle); + try + { + callbackHandle = GCHandle.Alloc(callback); + + IntPtr p = SherpaOnnxOfflineTtsGenerateWithProgressCallback( + _handle.Handle, + utf8BytesWithNull, + speakerId, + speed, + callback + ); + + return new OfflineTtsGeneratedAudio(p); + } + finally + { + if (callbackHandle.IsAllocated) + callbackHandle.Free(); + } } - public OfflineTtsGeneratedAudio GenerateWithConfig(string text, OfflineTtsGenerationConfig config, OfflineTtsCallbackProgressWithArg callback) + + public OfflineTtsGeneratedAudio GenerateWithConfig( + string text, + OfflineTtsGenerationConfig config, + OfflineTtsCallbackProgressWithArg callback) { - _callbackWithArgRef = callback; byte[] utf8Bytes = Encoding.UTF8.GetBytes(text); - byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; // +1 for null terminator + byte[] utf8BytesWithNull = new byte[utf8Bytes.Length + 1]; Array.Copy(utf8Bytes, utf8BytesWithNull, utf8Bytes.Length); + utf8BytesWithNull[utf8Bytes.Length] = 0; + GCHandle callbackHandle = default(GCHandle); GCHandle? audioHandle = null; - IntPtr p; var nativeConfig = config.ToNative(out audioHandle); try { - p = SherpaOnnxOfflineTtsGenerateWithConfig(_handle.Handle, utf8BytesWithNull, ref nativeConfig, _callbackWithArgRef, IntPtr.Zero); + callbackHandle = GCHandle.Alloc(callback); + + IntPtr p = SherpaOnnxOfflineTtsGenerateWithConfig( + _handle.Handle, + utf8BytesWithNull, + ref nativeConfig, + callback, + IntPtr.Zero + ); + + return new OfflineTtsGeneratedAudio(p); } finally { - // Ensure the pinned handle is freed even if P/Invoke throws + if (callbackHandle.IsAllocated) + callbackHandle.Free(); + if (audioHandle.HasValue) audioHandle.Value.Free(); } - - _callbackWithArgRef = null; // safe: blocking call - return new OfflineTtsGeneratedAudio(p); } - public void Dispose() { Cleanup(); diff --git a/scripts/dotnet/OfflineTtsGenerationConfig.cs b/scripts/dotnet/OfflineTtsGenerationConfig.cs index 390ac25db3..1ab12c4290 100644 --- a/scripts/dotnet/OfflineTtsGenerationConfig.cs +++ b/scripts/dotnet/OfflineTtsGenerationConfig.cs @@ -100,19 +100,14 @@ internal NativeStruct ToNative(out GCHandle? audioHandle) native.Extra = json.ToString(); } #else - if (Extra != null && Extra.Count > 0) - { - native.Extra = JsonSerializer.Serialize( - Extra, - new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }); - } - else - { - native.Extra = "{}"; - } + native.Extra = (Extra != null && Extra.Count > 0) + ? JsonSerializer.Serialize( + Extra, + new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }) + : "{}"; #endif return native;