diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 61a44c9333..5a75a4d77e 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -886,6 +886,7 @@ type OfflineTtsPocketModelConfig struct { TextConditioner string // text_conditioner VocabJson string // vocab_json TokenScoresJson string // token_scores_json + VoiceEmbeddingCacheCapacity int // voice_embedding_cache_capacity } type OfflineTtsZipvoiceModelConfig struct { @@ -1170,6 +1171,8 @@ func NewOfflineTts(config *OfflineTtsConfig) *OfflineTts { c.model.pocket.token_scores_json = C.CString(config.Model.Pocket.TokenScoresJson) defer C.free(unsafe.Pointer(c.model.pocket.token_scores_json)) + c.model.pocket.voice_embedding_cache_capacity = C.int(config.Model.Pocket.VoiceEmbeddingCacheCapacity) + c.model.num_threads = C.int(config.Model.NumThreads) c.model.debug = C.int(config.Model.Debug) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index d980554732..b7ca93572b 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -762,15 +762,14 @@ const SherpaOnnxOfflineRecognizerResult *SherpaOnnxGetOfflineStreamResult( // Copy segment-level timestamps (from Whisper with segment timestamps) auto segment_count = result.segment_texts.size(); - if (segment_count > 0 && - result.segment_timestamps.size() == segment_count && + if (segment_count > 0 && result.segment_timestamps.size() == segment_count && result.segment_durations.size() == segment_count) { r->segment_count = segment_count; // Copy segment timestamps float *timestamps = new float[segment_count]; - std::copy(result.segment_timestamps.begin(), result.segment_timestamps.end(), - timestamps); + std::copy(result.segment_timestamps.begin(), + result.segment_timestamps.end(), timestamps); r->segment_timestamps = timestamps; // Copy segment durations @@ -1363,6 +1362,12 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( SHERPA_ONNX_OR(config->model.pocket.vocab_json, ""); tts_config.model.pocket.token_scores_json = SHERPA_ONNX_OR(config->model.pocket.token_scores_json, ""); + if (config->model.pocket.voice_embedding_cache_capacity >= 0) { + tts_config.model.pocket.voice_embedding_cache_capacity = + config->model.pocket.voice_embedding_cache_capacity; + } else { + tts_config.model.pocket.voice_embedding_cache_capacity = 50; + } tts_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1); tts_config.model.debug = config->model.debug; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 50178b3330..b6c4436712 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1139,6 +1139,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsPocketModelConfig { const char *text_conditioner; const char *vocab_json; const char *token_scores_json; + int32_t voice_embedding_cache_capacity; } SherpaOnnxOfflineTtsPocketModelConfig; SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsModelConfig { diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 9640c3dd89..7992f74271 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -467,6 +467,9 @@ OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) { c.model.pocket.token_scores_json = config.model.pocket.token_scores_json.c_str(); + c.model.pocket.voice_embedding_cache_capacity = + config.model.pocket.voice_embedding_cache_capacity; + c.model.num_threads = config.model.num_threads; c.model.debug = config.model.debug; c.model.provider = config.model.provider.c_str(); diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 448401107b..b3f1dfb142 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -472,6 +472,7 @@ struct OfflineTtsPocketModelConfig { std::string vocab_json; std::string token_scores_json; + int32_t voice_embedding_cache_capacity = 50; }; struct OfflineTtsModelConfig { diff --git a/sherpa-onnx/csrc/normal-data-generator.cc b/sherpa-onnx/csrc/normal-data-generator.cc index 6c31863c7d..b6a1951966 100644 --- a/sherpa-onnx/csrc/normal-data-generator.cc +++ b/sherpa-onnx/csrc/normal-data-generator.cc @@ -10,6 +10,7 @@ namespace sherpa_onnx { // Helper type hidden in translation unit +namespace { struct RNGHolder { std::mt19937 rng; std::normal_distribution dist; @@ -24,20 +25,34 @@ struct RNGHolder { }()), dist() {} }; - -NormalDataGenerator::NormalDataGenerator(float mean /* = 0.0f */, - float stddev /* = 1.0f */) - : mean_(mean), stddev_(stddev) {} +} // namespace + +NormalDataGenerator::NormalDataGenerator(float mean /* = 0.0f*/, + float stddev /* = 1.0f*/, + int32_t seed /* = -1*/) + : mean_(mean), stddev_(stddev), seed_(seed) { + if (seed_ >= 0) { + rng_.seed(static_cast(seed_)); + } +} void NormalDataGenerator::Fill(float *data, std::size_t size) const { - // One RNGHolder per thread - static thread_local RNGHolder holder; - - holder.dist.param( - std::normal_distribution::param_type(mean_, stddev_)); - - for (std::size_t i = 0; i < size; ++i) { - data[i] = holder.dist(holder.rng); + if (seed_ >= 0) { + // Deterministic mode: use instance-level RNG + std::normal_distribution dist(mean_, stddev_); + for (std::size_t i = 0; i < size; ++i) { + data[i] = dist(rng_); + } + } else { + // Original behavior: thread-local random device + static thread_local RNGHolder holder; + + holder.dist.param( + std::normal_distribution::param_type(mean_, stddev_)); + + for (std::size_t i = 0; i < size; ++i) { + data[i] = holder.dist(holder.rng); + } } } diff --git a/sherpa-onnx/csrc/normal-data-generator.h b/sherpa-onnx/csrc/normal-data-generator.h index 17601b4f18..fa34b76014 100644 --- a/sherpa-onnx/csrc/normal-data-generator.h +++ b/sherpa-onnx/csrc/normal-data-generator.h @@ -6,12 +6,15 @@ #define SHERPA_ONNX_CSRC_NORMAL_DATA_GENERATOR_H_ #include +#include +#include namespace sherpa_onnx { class NormalDataGenerator { public: - explicit NormalDataGenerator(float mean = 0.0f, float stddev = 1.0f); + explicit NormalDataGenerator(float mean = 0.0f, float stddev = 1.0f, + int32_t seed = -1); // Fill pre-allocated memory void Fill(float *data, std::size_t size) const; @@ -19,6 +22,8 @@ class NormalDataGenerator { private: float mean_; float stddev_; + int32_t seed_ = -1; // -1 = use thread-local random device (default) + mutable std::mt19937 rng_; // used if seed_ >= 0 }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 1a74034051..7e8284454c 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -5,14 +5,20 @@ #define SHERPA_ONNX_CSRC_OFFLINE_TTS_POCKET_IMPL_H_ #include +#include #include +#include +#include +#include #include #include #include +#include #include +#include #include -#include #include +#include #include #include @@ -37,6 +43,8 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { model_(std::make_unique(config.model)) { InitTokenizer(); + cache_.SetCapacity(config.model.pocket.voice_embedding_cache_capacity); + if (!config.rule_fsts.empty()) { std::vector files; SplitStringToVector(config.rule_fsts, ",", false, &files); @@ -92,6 +100,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { : config_(config), model_(std::make_unique(mgr, config.model)) { InitTokenizer(mgr); + cache_.SetCapacity(config.model.pocket.voice_embedding_cache_capacity); if (!config.rule_fsts.empty()) { std::vector files; @@ -159,6 +168,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { * - max_reference_audio_len, float, default 10, in seconds * - max_char_in_sentence, int, default 200 * - min_char_in_sentence, int, default 30 + * - seed, int, default -1 */ GeneratedAudio Generate( const std::string &_text, const GenerationConfig &gen_config, @@ -361,6 +371,24 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { return sentences; } + static size_t ComputeHash(const float *p, size_t n) { + size_t hash = 0; + + auto hash_combine = [](size_t &seed, size_t value) { + seed ^= value + 0x9e3779b97f4a7c15ull + (seed << 6) + (seed >> 2); + }; + + hash_combine(hash, n); + + for (size_t i = 0; i < n; ++i) { + uint32_t bits; + std::memcpy(&bits, &p[i], sizeof(float)); + hash_combine(hash, bits); + } + + return hash; + } + GeneratedAudio GenerateSingleSentence( const std::string &text, const GenerationConfig &gen_config, Ort::Value voice_embedding, bool &should_continue, @@ -398,8 +426,9 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { int32_t frames_after_eos = gen_config.GetExtraInt("frames_after_eos", 3); float temperature = gen_config.GetExtraFloat("temperature", 0.7f); float stddev = std::sqrt(temperature); + int32_t seed = gen_config.GetExtraInt("seed", -1); - NormalDataGenerator normal_gen(0, stddev); + NormalDataGenerator normal_gen(0, stddev, seed); std::vector noise(32, 0); std::array noise_shape = {1, 32}; @@ -564,22 +593,42 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { num_samples = gen_config.reference_audio.size(); } - // in seconds float max_reference_audio_len = gen_config.GetExtraFloat("max_reference_audio_len", 10); + // in seconds + int32_t max_len = static_cast(max_reference_audio_len * SampleRate()); if (num_samples > max_len) { - SHERPA_ONNX_LOGE( - "max_reference_audio_len is %.3f seconds. Given reference audio of " - "%.3f seconds. Only the first %.3f seconds are used", - max_reference_audio_len, num_samples * 1.0f / SampleRate(), - max_reference_audio_len); + if (config_.model.debug) { + SHERPA_ONNX_LOGE( + "max_reference_audio_len is %.3f seconds. Given reference audio of " + "%.3f seconds. Only the first %.3f seconds are used", + max_reference_audio_len, num_samples * 1.0f / SampleRate(), + max_reference_audio_len); + } num_samples = max_len; } + // Compute hash of reference audio for cache lookup + size_t audio_hash = ComputeHash(p_audio, num_samples); + + auto cached_embedding = cache_.Get(audio_hash); + if (cached_embedding) { + if (config_.model.debug) { + SHERPA_ONNX_LOGE("CACHE HIT: voice embedding (hash=%zu)", audio_hash); + } + // Create an owned tensor and copy data to avoid use-after-free + auto result = Ort::Value::CreateTensor( + model_->Allocator(), cached_embedding->second.data(), + cached_embedding->second.size()); + std::copy(cached_embedding->first.begin(), cached_embedding->first.end(), + result.GetTensorMutableData()); + return result; + } + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); @@ -587,7 +636,23 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { Ort::Value x = Ort::Value::CreateTensor(memory_info, const_cast(p_audio), num_samples, shape.data(), shape.size()); - return model_->RunMimiEncoder(std::move(x)); + + Ort::Value result = model_->RunMimiEncoder(std::move(x)); + + auto info = result.GetTensorTypeAndShapeInfo(); + auto result_shape = info.GetShape(); + size_t total = info.GetElementCount(); + const float *result_data = result.GetTensorData(); + + cache_.Put(audio_hash, std::vector(result_data, result_data + total), + std::move(result_shape)); + + if (config_.model.debug) { + SHERPA_ONNX_LOGE("CACHE MISS: cached embedding (hash=%zu, %zu floats)", + audio_hash, total); + } + + return result; } Ort::Value GetTextEmbedding(const std::string &text) const { @@ -688,6 +753,115 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { std::unique_ptr model_; std::vector> tn_list_; std::unique_ptr tokenizer_; + + // Shared Thread-Safe LRU Cache for Voice Embeddings + struct VoiceEmbeddingCache { + using Embedding = std::pair, std::vector>; + using EmbeddingPtr = std::shared_ptr; + + private: + using ListNode = std::pair; + using ListIt = std::list::iterator; + + mutable std::mutex mutex_; + size_t capacity_; + + // Front = most recently used + std::list lru_list_; + + // Key -> iterator into lru_list_ + std::unordered_map map_; + + public: + static constexpr size_t kDefaultCapacity = 50; + + explicit VoiceEmbeddingCache(size_t cap = kDefaultCapacity) + : capacity_(cap) {} + + EmbeddingPtr Get(size_t key) { + std::lock_guard lock(mutex_); + + auto it = map_.find(key); + if (it == map_.end()) { + return nullptr; // cache miss + } + + // Move to front (most recently used) + if (it->second != lru_list_.begin()) { + lru_list_.splice(lru_list_.begin(), lru_list_, it->second); + } + + return it->second->second; // copy shared_ptr + } + + void Put(size_t key, std::vector data, std::vector shape) { + std::lock_guard lock(mutex_); + + if (capacity_ == 0) { + return; + } + + auto it = map_.find(key); + + // If exists, update and move to front + if (it != map_.end()) { + it->second->second = + std::make_shared(std::move(data), std::move(shape)); + + if (it->second != lru_list_.begin()) { + lru_list_.splice(lru_list_.begin(), lru_list_, it->second); + } + return; + } + + // Evict if full + if (lru_list_.size() >= capacity_) { + auto &last = lru_list_.back(); + size_t last_key = last.first; + + map_.erase(last_key); + lru_list_.pop_back(); // shared_ptr released here + } + + // Insert new at front + lru_list_.emplace_front( + key, std::make_shared(std::move(data), std::move(shape))); + + map_[key] = lru_list_.begin(); + } + + void SetCapacity(int32_t cap) { + if (cap < 0) { + SHERPA_ONNX_LOGE( + "voice_embedding_cache_capacity must be >= 0. Given: %d", cap); + SHERPA_ONNX_EXIT(-1); + } + + std::lock_guard lock(mutex_); + capacity_ = cap; + + while (lru_list_.size() > capacity_) { + auto &last = lru_list_.back(); + size_t last_key = last.first; + + map_.erase(last_key); + lru_list_.pop_back(); + } + } + + size_t Size() const { + std::lock_guard lock(mutex_); + return lru_list_.size(); + } + + void Clear() { + std::lock_guard lock(mutex_); + map_.clear(); + lru_list_.clear(); + } + }; + + mutable VoiceEmbeddingCache cache_; }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc b/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc index ec2435b326..ce866bb3b9 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc @@ -23,6 +23,10 @@ void OfflineTtsPocketModelConfig::Register(ParseOptions *po) { "Path to PocketTTS vocab.json"); po->Register("pocket-token-scores-json", &token_scores_json, "Path to PocketTTS token_scores.json"); + po->Register("pocket-voice-embedding-cache-capacity", + &voice_embedding_cache_capacity, + "Capacity of the voice embedding cache (number of items). " + "Default: 50. 0 disables caching."); } bool OfflineTtsPocketModelConfig::Validate() const { @@ -99,6 +103,13 @@ bool OfflineTtsPocketModelConfig::Validate() const { return false; } + if (voice_embedding_cache_capacity < 0) { + SHERPA_ONNX_LOGE( + "voice_embedding_cache_capacity must be non-negative. Given: %d", + voice_embedding_cache_capacity); + return false; + } + return true; } @@ -112,7 +123,9 @@ std::string OfflineTtsPocketModelConfig::ToString() const { os << "decoder=\"" << decoder << "\", "; os << "text_conditioner=\"" << text_conditioner << "\", "; os << "vocab_json=\"" << vocab_json << "\", "; - os << "token_scores_json=\"" << token_scores_json << "\")"; + os << "token_scores_json=\"" << token_scores_json << "\", "; + os << "voice_embedding_cache_capacity=" << voice_embedding_cache_capacity + << ")"; return os.str(); } diff --git a/sherpa-onnx/csrc/offline-tts-pocket-model-config.h b/sherpa-onnx/csrc/offline-tts-pocket-model-config.h index 97a280731b..81015d0401 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-model-config.h @@ -22,6 +22,7 @@ struct OfflineTtsPocketModelConfig { std::string token_scores_json; OfflineTtsPocketModelConfig() = default; + int32_t voice_embedding_cache_capacity = 50; OfflineTtsPocketModelConfig(const std::string &lm_flow, const std::string &lm_main,