From 8edfdb628a3a7c594b9865e20e87319ce2b8be99 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Sun, 15 Feb 2026 15:13:51 +1100 Subject: [PATCH 01/10] [PocketTTS] Add seed support and voice embedding caching for consistent generation Introduce a thread-safe LRU cache for voice embeddings to skip redundant Mimi encoder runs: compute a sampled hash of reference audio, return cached Ort tensor on hit, and store embeddings on miss. Expose voice_embedding_cache_capacity through model config, C API and Go wrapper, and add CLI flag pocket-voice-embedding-cache-capacity. Add deterministic NormalDataGenerator constructor (seed-aware) and use the generation 'seed' option to produce reproducible noise; preserve original thread-local RNG when seed < 0. Update config ToString and defaults, add necessary headers and logging/timing for cache hits/misses. --- scripts/go/sherpa_onnx.go | 3 + sherpa-onnx/c-api/c-api.cc | 2 + sherpa-onnx/c-api/c-api.h | 1 + sherpa-onnx/csrc/normal-data-generator.cc | 32 ++-- sherpa-onnx/csrc/normal-data-generator.h | 5 + sherpa-onnx/csrc/offline-tts-pocket-impl.h | 137 +++++++++++++++++- .../csrc/offline-tts-pocket-model-config.cc | 8 +- .../csrc/offline-tts-pocket-model-config.h | 1 + 8 files changed, 177 insertions(+), 12 deletions(-) diff --git a/scripts/go/sherpa_onnx.go b/scripts/go/sherpa_onnx.go index 61a44c9333..5a75a4d77e 100644 --- a/scripts/go/sherpa_onnx.go +++ b/scripts/go/sherpa_onnx.go @@ -886,6 +886,7 @@ type OfflineTtsPocketModelConfig struct { TextConditioner string // text_conditioner VocabJson string // vocab_json TokenScoresJson string // token_scores_json + VoiceEmbeddingCacheCapacity int // voice_embedding_cache_capacity } type OfflineTtsZipvoiceModelConfig struct { @@ -1170,6 +1171,8 @@ func NewOfflineTts(config *OfflineTtsConfig) *OfflineTts { c.model.pocket.token_scores_json = C.CString(config.Model.Pocket.TokenScoresJson) defer C.free(unsafe.Pointer(c.model.pocket.token_scores_json)) + c.model.pocket.voice_embedding_cache_capacity = C.int(config.Model.Pocket.VoiceEmbeddingCacheCapacity) + c.model.num_threads = C.int(config.Model.NumThreads) c.model.debug = C.int(config.Model.Debug) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index d980554732..09e67706fa 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -1363,6 +1363,8 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( SHERPA_ONNX_OR(config->model.pocket.vocab_json, ""); tts_config.model.pocket.token_scores_json = SHERPA_ONNX_OR(config->model.pocket.token_scores_json, ""); + tts_config.model.pocket.voice_embedding_cache_capacity = + SHERPA_ONNX_OR(config->model.pocket.voice_embedding_cache_capacity, 50); tts_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1); tts_config.model.debug = config->model.debug; diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index 50178b3330..b6c4436712 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -1139,6 +1139,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsPocketModelConfig { const char *text_conditioner; const char *vocab_json; const char *token_scores_json; + int32_t voice_embedding_cache_capacity; } SherpaOnnxOfflineTtsPocketModelConfig; SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsModelConfig { diff --git a/sherpa-onnx/csrc/normal-data-generator.cc b/sherpa-onnx/csrc/normal-data-generator.cc index 6c31863c7d..1415b1dbcc 100644 --- a/sherpa-onnx/csrc/normal-data-generator.cc +++ b/sherpa-onnx/csrc/normal-data-generator.cc @@ -10,6 +10,7 @@ namespace sherpa_onnx { // Helper type hidden in translation unit +namespace { struct RNGHolder { std::mt19937 rng; std::normal_distribution dist; @@ -24,20 +25,33 @@ struct RNGHolder { }()), dist() {} }; +} // namespace NormalDataGenerator::NormalDataGenerator(float mean /* = 0.0f */, float stddev /* = 1.0f */) - : mean_(mean), stddev_(stddev) {} + : mean_(mean), stddev_(stddev), seed_(-1) {} -void NormalDataGenerator::Fill(float *data, std::size_t size) const { - // One RNGHolder per thread - static thread_local RNGHolder holder; - - holder.dist.param( - std::normal_distribution::param_type(mean_, stddev_)); +NormalDataGenerator::NormalDataGenerator(float mean, float stddev, int32_t seed) + : mean_(mean), stddev_(stddev), seed_(seed) {} - for (std::size_t i = 0; i < size; ++i) { - data[i] = holder.dist(holder.rng); +void NormalDataGenerator::Fill(float *data, std::size_t size) const { + if (seed_ >= 0) { + // Deterministic mode: use fixed seed for reproducible noise + std::mt19937 rng(static_cast(seed_)); + std::normal_distribution dist(mean_, stddev_); + for (std::size_t i = 0; i < size; ++i) { + data[i] = dist(rng); + } + } else { + // Original behavior: thread-local random device + static thread_local RNGHolder holder; + + holder.dist.param( + std::normal_distribution::param_type(mean_, stddev_)); + + for (std::size_t i = 0; i < size; ++i) { + data[i] = holder.dist(holder.rng); + } } } diff --git a/sherpa-onnx/csrc/normal-data-generator.h b/sherpa-onnx/csrc/normal-data-generator.h index 17601b4f18..1400b56be5 100644 --- a/sherpa-onnx/csrc/normal-data-generator.h +++ b/sherpa-onnx/csrc/normal-data-generator.h @@ -6,6 +6,7 @@ #define SHERPA_ONNX_CSRC_NORMAL_DATA_GENERATOR_H_ #include +#include namespace sherpa_onnx { @@ -13,12 +14,16 @@ class NormalDataGenerator { public: explicit NormalDataGenerator(float mean = 0.0f, float stddev = 1.0f); + // Seed-aware constructor: when seed >= 0, uses deterministic RNG + NormalDataGenerator(float mean, float stddev, int32_t seed); + // Fill pre-allocated memory void Fill(float *data, std::size_t size) const; private: float mean_; float stddev_; + int32_t seed_ = -1; // -1 = use thread-local random device (default) }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 1a74034051..b94c9dc7b0 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -5,14 +5,19 @@ #define SHERPA_ONNX_CSRC_OFFLINE_TTS_POCKET_IMPL_H_ #include +#include #include +#include #include #include #include +#include #include +#include #include #include #include +#include #include #include @@ -92,6 +97,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { : config_(config), model_(std::make_unique(mgr, config.model)) { InitTokenizer(mgr); + GetCache().SetCapacity(config.model.pocket.voice_embedding_cache_capacity); if (!config.rule_fsts.empty()) { std::vector files; @@ -398,8 +404,9 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { int32_t frames_after_eos = gen_config.GetExtraInt("frames_after_eos", 3); float temperature = gen_config.GetExtraFloat("temperature", 0.7f); float stddev = std::sqrt(temperature); + int32_t seed = gen_config.GetExtraInt("seed", -1); - NormalDataGenerator normal_gen(0, stddev); + NormalDataGenerator normal_gen(0, stddev, seed); std::vector noise(32, 0); std::array noise_shape = {1, 32}; @@ -534,6 +541,36 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { return Ort::Value{nullptr}; } + // Compute hash of reference audio for cache lookup (sample every 100th) + size_t audio_hash = gen_config.reference_audio.size(); + for (size_t i = 0; i < gen_config.reference_audio.size(); i += 100) { + audio_hash ^= std::hash{}(gen_config.reference_audio[i]) + + 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); + } + + // Check cache + // Check shared LRU cache + std::vector data_copy; + std::vector shape_copy; + if (GetCache().Get(audio_hash, data_copy, shape_copy)) { + auto cache_start = std::chrono::high_resolution_clock::now(); + auto memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); + + auto result = Ort::Value::CreateTensor( + memory_info, data_copy.data(), data_copy.size(), shape_copy.data(), + shape_copy.size()); + auto cache_us = + std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - cache_start) + .count(); + SHERPA_ONNX_LOGE( + "CACHE HIT: voice embedding (hash=%zu) returned in %lld us (Mimi " + "encoder SKIPPED)", + audio_hash, (long long)cache_us); + return result; + } + std::vector reference_audio; const float *p_audio; @@ -587,7 +624,27 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { Ort::Value x = Ort::Value::CreateTensor(memory_info, const_cast(p_audio), num_samples, shape.data(), shape.size()); - return model_->RunMimiEncoder(std::move(x)); + auto embed_start = std::chrono::high_resolution_clock::now(); + Ort::Value result = model_->RunMimiEncoder(std::move(x)); + auto embed_ms = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - embed_start) + .count(); + + // Cache the embedding in shared LRU cache + auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); + size_t total = 1; + for (auto s : result_shape) total *= s; + const float *result_data = result.GetTensorData(); + + GetCache().Put( + audio_hash, std::vector(result_data, result_data + total), + std::vector(result_shape.begin(), result_shape.end())); + SHERPA_ONNX_LOGE( + "CACHE MISS: Mimi encoder took %lld ms → cached embedding (hash=%zu, " + "%zu floats)", + (long long)embed_ms, audio_hash, total); + + return result; } Ort::Value GetTextEmbedding(const std::string &text) const { @@ -688,6 +745,82 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { std::unique_ptr model_; std::vector> tn_list_; std::unique_ptr tokenizer_; + + // Shared Thread-Safe LRU Cache for Voice Embeddings + struct VoiceEmbeddingCache { + std::mutex mutex; + size_t capacity; + std::list lru_list; // List of hashes, most recently used at front + std::unordered_map, std::vector>> + data; + std::unordered_map::iterator> map_iters; + + static constexpr size_t kDefaultCapacity = 50; + + VoiceEmbeddingCache(size_t cap = kDefaultCapacity) : capacity(cap) {} + + bool Get(size_t key, std::vector &out_data, + std::vector &out_shape) { + std::lock_guard lock(mutex); + auto it = data.find(key); + if (it == data.end()) { + return false; + } + + // Move to front of LRU list + lru_list.splice(lru_list.begin(), lru_list, map_iters[key]); + + out_data = it->second.first; + out_shape = it->second.second; + return true; + } + + void SetCapacity(size_t cap) { + std::lock_guard lock(mutex); + capacity = cap; + while (data.size() > capacity) { + size_t last = lru_list.back(); + data.erase(last); + map_iters.erase(last); + lru_list.pop_back(); + } + } + + void Put(size_t key, const std::vector &in_data, + const std::vector &in_shape) { + std::lock_guard lock(mutex); + if (capacity == 0) { + return; + } + + auto it = data.find(key); + if (it != data.end()) { + // Update existing entry + it->second = {in_data, in_shape}; + lru_list.splice(lru_list.begin(), lru_list, map_iters[key]); + return; + } + + // Evict if necessary + if (data.size() >= capacity && !lru_list.empty()) { + size_t last = lru_list.back(); + data.erase(last); + map_iters.erase(last); + lru_list.pop_back(); + } + + // Insert new entry + lru_list.push_front(key); + data[key] = {in_data, in_shape}; + map_iters[key] = lru_list.begin(); + } + }; + + static VoiceEmbeddingCache &GetCache() { + static VoiceEmbeddingCache cache; + return cache; + } }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc b/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc index ec2435b326..df672bc06c 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc @@ -23,6 +23,10 @@ void OfflineTtsPocketModelConfig::Register(ParseOptions *po) { "Path to PocketTTS vocab.json"); po->Register("pocket-token-scores-json", &token_scores_json, "Path to PocketTTS token_scores.json"); + po->Register("pocket-voice-embedding-cache-capacity", + &voice_embedding_cache_capacity, + "Capacity of the voice embedding cache (number of items). " + "Default: 50"); } bool OfflineTtsPocketModelConfig::Validate() const { @@ -112,7 +116,9 @@ std::string OfflineTtsPocketModelConfig::ToString() const { os << "decoder=\"" << decoder << "\", "; os << "text_conditioner=\"" << text_conditioner << "\", "; os << "vocab_json=\"" << vocab_json << "\", "; - os << "token_scores_json=\"" << token_scores_json << "\")"; + os << "token_scores_json=\"" << token_scores_json << "\", "; + os << "voice_embedding_cache_capacity=" << voice_embedding_cache_capacity + << ")"; return os.str(); } diff --git a/sherpa-onnx/csrc/offline-tts-pocket-model-config.h b/sherpa-onnx/csrc/offline-tts-pocket-model-config.h index 97a280731b..81015d0401 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-model-config.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-model-config.h @@ -22,6 +22,7 @@ struct OfflineTtsPocketModelConfig { std::string token_scores_json; OfflineTtsPocketModelConfig() = default; + int32_t voice_embedding_cache_capacity = 50; OfflineTtsPocketModelConfig(const std::string &lm_flow, const std::string &lm_main, From 6e279cc98d1f2dca3846b005495a3c3067c046ce Mon Sep 17 00:00:00 2001 From: Simon Law Date: Sun, 15 Feb 2026 17:55:32 +1100 Subject: [PATCH 02/10] Improve TTS pocket caching, RNG, and tensor handling Several fixes and improvements across offline TTS and utilities: - c-api: minor formatting cleanup when copying segment timestamps; handle pocket voice_embedding_cache_capacity explicitly (use provided non-negative value or default to 50). - normal-data-generator: switch to an instance-level std::mt19937 seeded in the constructor for deterministic mode and use it in Fill(); added and rng_ member. - offline-tts-pocket-impl.h: strengthen cache key by mixing in sample rate and length to reduce collisions; create an owned Ort tensor via AllocatorWithDefaultOptions and CreateTensor(), then copy cached data into the tensor to avoid potential use-after-free. - offline-tts-pocket-model-config.cc: update help text to note that 0 disables caching and add validation to reject negative voice_embedding_cache_capacity with an error log. These changes improve determinism, correctness of cached tensor lifetimes, and robustness of the cache key and configuration validation. --- sherpa-onnx/c-api/c-api.cc | 15 +++++++++------ sherpa-onnx/csrc/normal-data-generator.cc | 11 +++++++---- sherpa-onnx/csrc/normal-data-generator.h | 4 +++- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 17 ++++++++++++----- .../csrc/offline-tts-pocket-model-config.cc | 9 ++++++++- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 09e67706fa..b7ca93572b 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -762,15 +762,14 @@ const SherpaOnnxOfflineRecognizerResult *SherpaOnnxGetOfflineStreamResult( // Copy segment-level timestamps (from Whisper with segment timestamps) auto segment_count = result.segment_texts.size(); - if (segment_count > 0 && - result.segment_timestamps.size() == segment_count && + if (segment_count > 0 && result.segment_timestamps.size() == segment_count && result.segment_durations.size() == segment_count) { r->segment_count = segment_count; // Copy segment timestamps float *timestamps = new float[segment_count]; - std::copy(result.segment_timestamps.begin(), result.segment_timestamps.end(), - timestamps); + std::copy(result.segment_timestamps.begin(), + result.segment_timestamps.end(), timestamps); r->segment_timestamps = timestamps; // Copy segment durations @@ -1363,8 +1362,12 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig( SHERPA_ONNX_OR(config->model.pocket.vocab_json, ""); tts_config.model.pocket.token_scores_json = SHERPA_ONNX_OR(config->model.pocket.token_scores_json, ""); - tts_config.model.pocket.voice_embedding_cache_capacity = - SHERPA_ONNX_OR(config->model.pocket.voice_embedding_cache_capacity, 50); + if (config->model.pocket.voice_embedding_cache_capacity >= 0) { + tts_config.model.pocket.voice_embedding_cache_capacity = + config->model.pocket.voice_embedding_cache_capacity; + } else { + tts_config.model.pocket.voice_embedding_cache_capacity = 50; + } tts_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1); tts_config.model.debug = config->model.debug; diff --git a/sherpa-onnx/csrc/normal-data-generator.cc b/sherpa-onnx/csrc/normal-data-generator.cc index 1415b1dbcc..1dd992c45e 100644 --- a/sherpa-onnx/csrc/normal-data-generator.cc +++ b/sherpa-onnx/csrc/normal-data-generator.cc @@ -32,15 +32,18 @@ NormalDataGenerator::NormalDataGenerator(float mean /* = 0.0f */, : mean_(mean), stddev_(stddev), seed_(-1) {} NormalDataGenerator::NormalDataGenerator(float mean, float stddev, int32_t seed) - : mean_(mean), stddev_(stddev), seed_(seed) {} + : mean_(mean), stddev_(stddev), seed_(seed) { + if (seed_ >= 0) { + rng_.seed(static_cast(seed_)); + } +} void NormalDataGenerator::Fill(float *data, std::size_t size) const { if (seed_ >= 0) { - // Deterministic mode: use fixed seed for reproducible noise - std::mt19937 rng(static_cast(seed_)); + // Deterministic mode: use instance-level RNG std::normal_distribution dist(mean_, stddev_); for (std::size_t i = 0; i < size; ++i) { - data[i] = dist(rng); + data[i] = dist(rng_); } } else { // Original behavior: thread-local random device diff --git a/sherpa-onnx/csrc/normal-data-generator.h b/sherpa-onnx/csrc/normal-data-generator.h index 1400b56be5..b73ed401e7 100644 --- a/sherpa-onnx/csrc/normal-data-generator.h +++ b/sherpa-onnx/csrc/normal-data-generator.h @@ -7,6 +7,7 @@ #include #include +#include namespace sherpa_onnx { @@ -23,7 +24,8 @@ class NormalDataGenerator { private: float mean_; float stddev_; - int32_t seed_ = -1; // -1 = use thread-local random device (default) + int32_t seed_ = -1; // -1 = use thread-local random device (default) + mutable std::mt19937 rng_; // used if seed_ >= 0 }; } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index b94c9dc7b0..0c03d7b64f 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -542,7 +542,13 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { } // Compute hash of reference audio for cache lookup (sample every 100th) + // We mix in the sample rate and length to avoid collisions. size_t audio_hash = gen_config.reference_audio.size(); + audio_hash ^= std::hash{}(gen_config.reference_sample_rate) + + 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); + audio_hash ^= std::hash{}(gen_config.reference_audio.size()) + + 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); + for (size_t i = 0; i < gen_config.reference_audio.size(); i += 100) { audio_hash ^= std::hash{}(gen_config.reference_audio[i]) + 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); @@ -554,12 +560,13 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { std::vector shape_copy; if (GetCache().Get(audio_hash, data_copy, shape_copy)) { auto cache_start = std::chrono::high_resolution_clock::now(); - auto memory_info = - Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); - auto result = Ort::Value::CreateTensor( - memory_info, data_copy.data(), data_copy.size(), shape_copy.data(), - shape_copy.size()); + // Create an owned tensor and copy data to avoid use-after-free + Ort::AllocatorWithDefaultOptions allocator; + auto result = Ort::Value::CreateTensor( + allocator, shape_copy.data(), shape_copy.size()); + std::copy(data_copy.begin(), data_copy.end(), + result.GetTensorMutableData()); auto cache_us = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - cache_start) diff --git a/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc b/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc index df672bc06c..ce866bb3b9 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc +++ b/sherpa-onnx/csrc/offline-tts-pocket-model-config.cc @@ -26,7 +26,7 @@ void OfflineTtsPocketModelConfig::Register(ParseOptions *po) { po->Register("pocket-voice-embedding-cache-capacity", &voice_embedding_cache_capacity, "Capacity of the voice embedding cache (number of items). " - "Default: 50"); + "Default: 50. 0 disables caching."); } bool OfflineTtsPocketModelConfig::Validate() const { @@ -103,6 +103,13 @@ bool OfflineTtsPocketModelConfig::Validate() const { return false; } + if (voice_embedding_cache_capacity < 0) { + SHERPA_ONNX_LOGE( + "voice_embedding_cache_capacity must be non-negative. Given: %d", + voice_embedding_cache_capacity); + return false; + } + return true; } From d7e9f45957bd0f375d83e051e927f03be096f266 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Sun, 15 Feb 2026 18:24:39 +1100 Subject: [PATCH 03/10] Use accumulate and move vectors in cache Replace manual product loop with std::accumulate + std::multiplies for computing tensor element count. Change Cache::Put to take vectors by value and std::move them into the map (both on update and insert) to avoid unnecessary copies and improve performance. --- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 0c03d7b64f..31aa9b0906 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -639,8 +639,8 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { // Cache the embedding in shared LRU cache auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); - size_t total = 1; - for (auto s : result_shape) total *= s; + size_t total = std::accumulate(result_shape.begin(), result_shape.end(), 1, + std::multiplies()); const float *result_data = result.GetTensorData(); GetCache().Put( @@ -794,8 +794,8 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { } } - void Put(size_t key, const std::vector &in_data, - const std::vector &in_shape) { + void Put(size_t key, std::vector in_data, + std::vector in_shape) { std::lock_guard lock(mutex); if (capacity == 0) { return; @@ -804,7 +804,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { auto it = data.find(key); if (it != data.end()) { // Update existing entry - it->second = {in_data, in_shape}; + it->second = {std::move(in_data), std::move(in_shape)}; lru_list.splice(lru_list.begin(), lru_list, map_iters[key]); return; } @@ -819,7 +819,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { // Insert new entry lru_list.push_front(key); - data[key] = {in_data, in_shape}; + data[key] = {std::move(in_data), std::move(in_shape)}; map_iters[key] = lru_list.begin(); } }; From e804cc160336773250fbb52a931a875b1703e03e Mon Sep 17 00:00:00 2001 From: Simon Law Date: Mon, 16 Feb 2026 18:50:36 +1100 Subject: [PATCH 04/10] Refine cache hashing and improve logging - Robustify voice embedding cache key: include `max_reference_audio_len` and hash all reference audio samples to prevent collisions. - Optimization: Switch cache hit/miss logs from ERROR to INFO and make timing logic conditional on debug mode. - Refactor: Merge [NormalDataGenerator] constructors and remove unused `` header. --- sherpa-onnx/c-api/cxx-api.cc | 3 + sherpa-onnx/c-api/cxx-api.h | 1 + sherpa-onnx/csrc/normal-data-generator.cc | 4 - sherpa-onnx/csrc/normal-data-generator.h | 6 +- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 142 +++++++++++++-------- 5 files changed, 96 insertions(+), 60 deletions(-) diff --git a/sherpa-onnx/c-api/cxx-api.cc b/sherpa-onnx/c-api/cxx-api.cc index 9640c3dd89..7992f74271 100644 --- a/sherpa-onnx/c-api/cxx-api.cc +++ b/sherpa-onnx/c-api/cxx-api.cc @@ -467,6 +467,9 @@ OfflineTts OfflineTts::Create(const OfflineTtsConfig &config) { c.model.pocket.token_scores_json = config.model.pocket.token_scores_json.c_str(); + c.model.pocket.voice_embedding_cache_capacity = + config.model.pocket.voice_embedding_cache_capacity; + c.model.num_threads = config.model.num_threads; c.model.debug = config.model.debug; c.model.provider = config.model.provider.c_str(); diff --git a/sherpa-onnx/c-api/cxx-api.h b/sherpa-onnx/c-api/cxx-api.h index 448401107b..b3f1dfb142 100644 --- a/sherpa-onnx/c-api/cxx-api.h +++ b/sherpa-onnx/c-api/cxx-api.h @@ -472,6 +472,7 @@ struct OfflineTtsPocketModelConfig { std::string vocab_json; std::string token_scores_json; + int32_t voice_embedding_cache_capacity = 50; }; struct OfflineTtsModelConfig { diff --git a/sherpa-onnx/csrc/normal-data-generator.cc b/sherpa-onnx/csrc/normal-data-generator.cc index 1dd992c45e..2ace7fa9a4 100644 --- a/sherpa-onnx/csrc/normal-data-generator.cc +++ b/sherpa-onnx/csrc/normal-data-generator.cc @@ -27,10 +27,6 @@ struct RNGHolder { }; } // namespace -NormalDataGenerator::NormalDataGenerator(float mean /* = 0.0f */, - float stddev /* = 1.0f */) - : mean_(mean), stddev_(stddev), seed_(-1) {} - NormalDataGenerator::NormalDataGenerator(float mean, float stddev, int32_t seed) : mean_(mean), stddev_(stddev), seed_(seed) { if (seed_ >= 0) { diff --git a/sherpa-onnx/csrc/normal-data-generator.h b/sherpa-onnx/csrc/normal-data-generator.h index b73ed401e7..fa34b76014 100644 --- a/sherpa-onnx/csrc/normal-data-generator.h +++ b/sherpa-onnx/csrc/normal-data-generator.h @@ -13,10 +13,8 @@ namespace sherpa_onnx { class NormalDataGenerator { public: - explicit NormalDataGenerator(float mean = 0.0f, float stddev = 1.0f); - - // Seed-aware constructor: when seed >= 0, uses deterministic RNG - NormalDataGenerator(float mean, float stddev, int32_t seed); + explicit NormalDataGenerator(float mean = 0.0f, float stddev = 1.0f, + int32_t seed = -1); // Fill pre-allocated memory void Fill(float *data, std::size_t size) const; diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 31aa9b0906..083e412a2a 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -25,6 +24,7 @@ #include "kaldifst/csrc/kaldi-fst-io.h" #include "kaldifst/csrc/text-normalizer.h" #include "sherpa-onnx/csrc/file-utils.h" +#include "sherpa-onnx/csrc/log.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/math.h" #include "sherpa-onnx/csrc/normal-data-generator.h" @@ -42,6 +42,8 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { model_(std::make_unique(config.model)) { InitTokenizer(); + cache_.SetCapacity(config.model.pocket.voice_embedding_cache_capacity); + if (!config.rule_fsts.empty()) { std::vector files; SplitStringToVector(config.rule_fsts, ",", false, &files); @@ -97,7 +99,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { : config_(config), model_(std::make_unique(mgr, config.model)) { InitTokenizer(mgr); - GetCache().SetCapacity(config.model.pocket.voice_embedding_cache_capacity); + cache_.SetCapacity(config.model.pocket.voice_embedding_cache_capacity); if (!config.rule_fsts.empty()) { std::vector files; @@ -549,33 +551,50 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { audio_hash ^= std::hash{}(gen_config.reference_audio.size()) + 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); - for (size_t i = 0; i < gen_config.reference_audio.size(); i += 100) { - audio_hash ^= std::hash{}(gen_config.reference_audio[i]) + - 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); + for (float s : gen_config.reference_audio) { + audio_hash ^= std::hash{}(s) + 0x9e3779b9 + (audio_hash << 6) + + (audio_hash >> 2); } + // Include max_reference_audio_len in hash to distinguish between same audio + // but different truncation settings. + float max_reference_audio_len = + gen_config.GetExtraFloat("max_reference_audio_len", 10); + audio_hash ^= std::hash{}(max_reference_audio_len) + 0x9e3779b9 + + (audio_hash << 6) + (audio_hash >> 2); + // Check cache // Check shared LRU cache std::vector data_copy; std::vector shape_copy; - if (GetCache().Get(audio_hash, data_copy, shape_copy)) { - auto cache_start = std::chrono::high_resolution_clock::now(); - - // Create an owned tensor and copy data to avoid use-after-free - Ort::AllocatorWithDefaultOptions allocator; - auto result = Ort::Value::CreateTensor( - allocator, shape_copy.data(), shape_copy.size()); - std::copy(data_copy.begin(), data_copy.end(), - result.GetTensorMutableData()); - auto cache_us = - std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - cache_start) - .count(); - SHERPA_ONNX_LOGE( - "CACHE HIT: voice embedding (hash=%zu) returned in %lld us (Mimi " - "encoder SKIPPED)", - audio_hash, (long long)cache_us); - return result; + if (cache_.Get(audio_hash, data_copy, shape_copy)) { + if (config_.model.debug) { + auto cache_start = std::chrono::high_resolution_clock::now(); + + // Create an owned tensor and copy data to avoid use-after-free + Ort::AllocatorWithDefaultOptions allocator; + auto result = Ort::Value::CreateTensor( + allocator, shape_copy.data(), shape_copy.size()); + std::copy(data_copy.begin(), data_copy.end(), + result.GetTensorMutableData()); + auto cache_us = + std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - cache_start) + .count(); + SHERPA_ONNX_LOG(INFO) + << "CACHE HIT: voice embedding (hash=" << audio_hash + << ") returned in " << (long long)cache_us + << " us (Mimi encoder SKIPPED)"; + return result; + } else { + // Create an owned tensor and copy data to avoid use-after-free + Ort::AllocatorWithDefaultOptions allocator; + auto result = Ort::Value::CreateTensor( + allocator, shape_copy.data(), shape_copy.size()); + std::copy(data_copy.begin(), data_copy.end(), + result.GetTensorMutableData()); + return result; + } } std::vector reference_audio; @@ -609,18 +628,20 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { } // in seconds - float max_reference_audio_len = - gen_config.GetExtraFloat("max_reference_audio_len", 10); + // max_reference_audio_len is already computed above for the hash. int32_t max_len = static_cast(max_reference_audio_len * SampleRate()); if (num_samples > max_len) { - SHERPA_ONNX_LOGE( - "max_reference_audio_len is %.3f seconds. Given reference audio of " - "%.3f seconds. Only the first %.3f seconds are used", - max_reference_audio_len, num_samples * 1.0f / SampleRate(), - max_reference_audio_len); + if (config_.model.debug) { + SHERPA_ONNX_LOG(INFO) + << "max_reference_audio_len is " << max_reference_audio_len + << " seconds. Given reference audio of " + << (num_samples * 1.0f / SampleRate()) + << " seconds. Only the first " << max_reference_audio_len + << " seconds are used"; + } num_samples = max_len; } @@ -631,25 +652,45 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { Ort::Value x = Ort::Value::CreateTensor(memory_info, const_cast(p_audio), num_samples, shape.data(), shape.size()); - auto embed_start = std::chrono::high_resolution_clock::now(); + + std::chrono::time_point embed_start; + if (config_.model.debug) { + embed_start = std::chrono::high_resolution_clock::now(); + } + Ort::Value result = model_->RunMimiEncoder(std::move(x)); - auto embed_ms = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - embed_start) - .count(); - - // Cache the embedding in shared LRU cache - auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); - size_t total = std::accumulate(result_shape.begin(), result_shape.end(), 1, - std::multiplies()); - const float *result_data = result.GetTensorData(); - - GetCache().Put( - audio_hash, std::vector(result_data, result_data + total), - std::vector(result_shape.begin(), result_shape.end())); - SHERPA_ONNX_LOGE( - "CACHE MISS: Mimi encoder took %lld ms → cached embedding (hash=%zu, " - "%zu floats)", - (long long)embed_ms, audio_hash, total); + + if (config_.model.debug) { + auto embed_ms = + std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - embed_start) + .count(); + + // Cache the embedding in shared LRU cache + auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); + size_t total = std::accumulate(result_shape.begin(), result_shape.end(), + 1, std::multiplies()); + const float *result_data = result.GetTensorData(); + + cache_.Put( + audio_hash, std::vector(result_data, result_data + total), + std::vector(result_shape.begin(), result_shape.end())); + + SHERPA_ONNX_LOG(INFO) + << "CACHE MISS: Mimi encoder took " << (long long)embed_ms + << " ms → cached embedding (hash=" << audio_hash << ", " << total + << " floats)"; + } else { + // Cache the embedding in shared LRU cache (no logging) + auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); + size_t total = std::accumulate(result_shape.begin(), result_shape.end(), + 1, std::multiplies()); + const float *result_data = result.GetTensorData(); + + cache_.Put( + audio_hash, std::vector(result_data, result_data + total), + std::vector(result_shape.begin(), result_shape.end())); + } return result; } @@ -824,10 +865,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { } }; - static VoiceEmbeddingCache &GetCache() { - static VoiceEmbeddingCache cache; - return cache; - } + mutable VoiceEmbeddingCache cache_; }; } // namespace sherpa_onnx From 23cdfce4cf4eab50f11f3ed8b7bfc860ce272cca Mon Sep 17 00:00:00 2001 From: Simon Law Date: Wed, 18 Feb 2026 09:01:27 +1100 Subject: [PATCH 05/10] Apply suggestion from @csukuangfj Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/normal-data-generator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/normal-data-generator.cc b/sherpa-onnx/csrc/normal-data-generator.cc index 2ace7fa9a4..69313fa034 100644 --- a/sherpa-onnx/csrc/normal-data-generator.cc +++ b/sherpa-onnx/csrc/normal-data-generator.cc @@ -27,7 +27,7 @@ struct RNGHolder { }; } // namespace -NormalDataGenerator::NormalDataGenerator(float mean, float stddev, int32_t seed) +NormalDataGenerator::NormalDataGenerator(float mean = /*0.0f*/, float stddev = /*1.0f*/, int32_t seed = /*-1*/) : mean_(mean), stddev_(stddev), seed_(seed) { if (seed_ >= 0) { rng_.seed(static_cast(seed_)); From 6de339805d27376625ab09706f525ce34d673c9c Mon Sep 17 00:00:00 2001 From: Simon Law Date: Wed, 18 Feb 2026 10:19:28 +1100 Subject: [PATCH 06/10] addressed the PR feedback for the cache logic and improving code quality --- sherpa-onnx/csrc/normal-data-generator.cc | 4 +- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 257 +++++++++++---------- 2 files changed, 137 insertions(+), 124 deletions(-) diff --git a/sherpa-onnx/csrc/normal-data-generator.cc b/sherpa-onnx/csrc/normal-data-generator.cc index 69313fa034..b6a1951966 100644 --- a/sherpa-onnx/csrc/normal-data-generator.cc +++ b/sherpa-onnx/csrc/normal-data-generator.cc @@ -27,7 +27,9 @@ struct RNGHolder { }; } // namespace -NormalDataGenerator::NormalDataGenerator(float mean = /*0.0f*/, float stddev = /*1.0f*/, int32_t seed = /*-1*/) +NormalDataGenerator::NormalDataGenerator(float mean /* = 0.0f*/, + float stddev /* = 1.0f*/, + int32_t seed /* = -1*/) : mean_(mean), stddev_(stddev), seed_(seed) { if (seed_ >= 0) { rng_.seed(static_cast(seed_)); diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 083e412a2a..8db19f5593 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -24,7 +24,6 @@ #include "kaldifst/csrc/kaldi-fst-io.h" #include "kaldifst/csrc/text-normalizer.h" #include "sherpa-onnx/csrc/file-utils.h" -#include "sherpa-onnx/csrc/log.h" #include "sherpa-onnx/csrc/macros.h" #include "sherpa-onnx/csrc/math.h" #include "sherpa-onnx/csrc/normal-data-generator.h" @@ -167,6 +166,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { * - max_reference_audio_len, float, default 10, in seconds * - max_char_in_sentence, int, default 200 * - min_char_in_sentence, int, default 30 + * - seed, int, default -1 */ GeneratedAudio Generate( const std::string &_text, const GenerationConfig &gen_config, @@ -369,6 +369,15 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { return sentences; } + static size_t ComputeHash(const float *p, int32_t n) { + size_t hash = n; + hash ^= std::hash{}(n) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + for (int32_t i = 0; i < n; ++i) { + hash ^= std::hash{}(p[i]) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + } + return hash; + } + GeneratedAudio GenerateSingleSentence( const std::string &text, const GenerationConfig &gen_config, Ort::Value voice_embedding, bool &should_continue, @@ -543,60 +552,6 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { return Ort::Value{nullptr}; } - // Compute hash of reference audio for cache lookup (sample every 100th) - // We mix in the sample rate and length to avoid collisions. - size_t audio_hash = gen_config.reference_audio.size(); - audio_hash ^= std::hash{}(gen_config.reference_sample_rate) + - 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); - audio_hash ^= std::hash{}(gen_config.reference_audio.size()) + - 0x9e3779b9 + (audio_hash << 6) + (audio_hash >> 2); - - for (float s : gen_config.reference_audio) { - audio_hash ^= std::hash{}(s) + 0x9e3779b9 + (audio_hash << 6) + - (audio_hash >> 2); - } - - // Include max_reference_audio_len in hash to distinguish between same audio - // but different truncation settings. - float max_reference_audio_len = - gen_config.GetExtraFloat("max_reference_audio_len", 10); - audio_hash ^= std::hash{}(max_reference_audio_len) + 0x9e3779b9 + - (audio_hash << 6) + (audio_hash >> 2); - - // Check cache - // Check shared LRU cache - std::vector data_copy; - std::vector shape_copy; - if (cache_.Get(audio_hash, data_copy, shape_copy)) { - if (config_.model.debug) { - auto cache_start = std::chrono::high_resolution_clock::now(); - - // Create an owned tensor and copy data to avoid use-after-free - Ort::AllocatorWithDefaultOptions allocator; - auto result = Ort::Value::CreateTensor( - allocator, shape_copy.data(), shape_copy.size()); - std::copy(data_copy.begin(), data_copy.end(), - result.GetTensorMutableData()); - auto cache_us = - std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - cache_start) - .count(); - SHERPA_ONNX_LOG(INFO) - << "CACHE HIT: voice embedding (hash=" << audio_hash - << ") returned in " << (long long)cache_us - << " us (Mimi encoder SKIPPED)"; - return result; - } else { - // Create an owned tensor and copy data to avoid use-after-free - Ort::AllocatorWithDefaultOptions allocator; - auto result = Ort::Value::CreateTensor( - allocator, shape_copy.data(), shape_copy.size()); - std::copy(data_copy.begin(), data_copy.end(), - result.GetTensorMutableData()); - return result; - } - } - std::vector reference_audio; const float *p_audio; @@ -627,6 +582,45 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { num_samples = gen_config.reference_audio.size(); } + // Compute hash of reference audio for cache lookup + size_t audio_hash = ComputeHash(p_audio, num_samples); + + // Include max_reference_audio_len in hash to distinguish between same audio + // but different truncation settings. + float max_reference_audio_len = + gen_config.GetExtraFloat("max_reference_audio_len", 10); + audio_hash ^= std::hash{}(max_reference_audio_len) + 0x9e3779b9 + + (audio_hash << 6) + (audio_hash >> 2); + + // Check cache + // Check shared LRU cache + auto cached_embedding = cache_.Get(audio_hash); + if (cached_embedding) { + if (config_.model.debug) { + // Create an owned tensor and copy data to avoid use-after-free + Ort::AllocatorWithDefaultOptions allocator; + auto result = Ort::Value::CreateTensor( + allocator, cached_embedding->second.data(), + cached_embedding->second.size()); + std::copy(cached_embedding->first.begin(), + cached_embedding->first.end(), + result.GetTensorMutableData()); + + SHERPA_ONNX_LOGE("CACHE HIT: voice embedding (hash=%zu)", audio_hash); + return result; + } else { + // Create an owned tensor and copy data to avoid use-after-free + Ort::AllocatorWithDefaultOptions allocator; + auto result = Ort::Value::CreateTensor( + allocator, cached_embedding->second.data(), + cached_embedding->second.size()); + std::copy(cached_embedding->first.begin(), + cached_embedding->first.end(), + result.GetTensorMutableData()); + return result; + } + } + // in seconds // max_reference_audio_len is already computed above for the hash. @@ -635,12 +629,11 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { if (num_samples > max_len) { if (config_.model.debug) { - SHERPA_ONNX_LOG(INFO) - << "max_reference_audio_len is " << max_reference_audio_len - << " seconds. Given reference audio of " - << (num_samples * 1.0f / SampleRate()) - << " seconds. Only the first " << max_reference_audio_len - << " seconds are used"; + SHERPA_ONNX_LOGE( + "max_reference_audio_len is %.3f seconds. Given reference audio of " + "%.3f seconds. Only the first %.3f seconds are used", + max_reference_audio_len, num_samples * 1.0f / SampleRate(), + max_reference_audio_len); } num_samples = max_len; } @@ -653,19 +646,9 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { Ort::Value::CreateTensor(memory_info, const_cast(p_audio), num_samples, shape.data(), shape.size()); - std::chrono::time_point embed_start; - if (config_.model.debug) { - embed_start = std::chrono::high_resolution_clock::now(); - } - Ort::Value result = model_->RunMimiEncoder(std::move(x)); if (config_.model.debug) { - auto embed_ms = - std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - embed_start) - .count(); - // Cache the embedding in shared LRU cache auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); size_t total = std::accumulate(result_shape.begin(), result_shape.end(), @@ -676,10 +659,8 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { audio_hash, std::vector(result_data, result_data + total), std::vector(result_shape.begin(), result_shape.end())); - SHERPA_ONNX_LOG(INFO) - << "CACHE MISS: Mimi encoder took " << (long long)embed_ms - << " ms → cached embedding (hash=" << audio_hash << ", " << total - << " floats)"; + SHERPA_ONNX_LOGE("CACHE MISS: cached embedding (hash=%zu, %zu floats)", + audio_hash, total); } else { // Cache the embedding in shared LRU cache (no logging) auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); @@ -796,72 +777,102 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { // Shared Thread-Safe LRU Cache for Voice Embeddings struct VoiceEmbeddingCache { - std::mutex mutex; - size_t capacity; - std::list lru_list; // List of hashes, most recently used at front - std::unordered_map, std::vector>> - data; - std::unordered_map::iterator> map_iters; + using Embedding = std::pair, std::vector>; + using EmbeddingPtr = std::shared_ptr; + + private: + using ListNode = std::pair; + using ListIt = std::list::iterator; + + mutable std::mutex mutex_; + size_t capacity_; + + // Front = most recently used + std::list lru_list_; + // Key -> iterator into lru_list_ + std::unordered_map map_; + + public: static constexpr size_t kDefaultCapacity = 50; - VoiceEmbeddingCache(size_t cap = kDefaultCapacity) : capacity(cap) {} + explicit VoiceEmbeddingCache(size_t cap = kDefaultCapacity) + : capacity_(cap) {} + + EmbeddingPtr Get(size_t key) { + std::lock_guard lock(mutex_); - bool Get(size_t key, std::vector &out_data, - std::vector &out_shape) { - std::lock_guard lock(mutex); - auto it = data.find(key); - if (it == data.end()) { - return false; + auto it = map_.find(key); + if (it == map_.end()) { + return nullptr; // cache miss } - // Move to front of LRU list - lru_list.splice(lru_list.begin(), lru_list, map_iters[key]); + // Move to front (most recently used) + if (it->second != lru_list_.begin()) { + lru_list_.splice(lru_list_.begin(), lru_list_, it->second); + } - out_data = it->second.first; - out_shape = it->second.second; - return true; + return it->second->second; // copy shared_ptr } - void SetCapacity(size_t cap) { - std::lock_guard lock(mutex); - capacity = cap; - while (data.size() > capacity) { - size_t last = lru_list.back(); - data.erase(last); - map_iters.erase(last); - lru_list.pop_back(); - } - } + void Put(size_t key, std::vector data, std::vector shape) { + std::lock_guard lock(mutex_); - void Put(size_t key, std::vector in_data, - std::vector in_shape) { - std::lock_guard lock(mutex); - if (capacity == 0) { + if (capacity_ == 0) { return; } - auto it = data.find(key); - if (it != data.end()) { - // Update existing entry - it->second = {std::move(in_data), std::move(in_shape)}; - lru_list.splice(lru_list.begin(), lru_list, map_iters[key]); + auto it = map_.find(key); + + // If exists, update and move to front + if (it != map_.end()) { + it->second->second = + std::make_shared(std::move(data), std::move(shape)); + + if (it->second != lru_list_.begin()) { + lru_list_.splice(lru_list_.begin(), lru_list_, it->second); + } return; } - // Evict if necessary - if (data.size() >= capacity && !lru_list.empty()) { - size_t last = lru_list.back(); - data.erase(last); - map_iters.erase(last); - lru_list.pop_back(); + // Evict if full + if (lru_list_.size() >= capacity_) { + auto &last = lru_list_.back(); + size_t last_key = last.first; + + map_.erase(last_key); + lru_list_.pop_back(); // shared_ptr released here } - // Insert new entry - lru_list.push_front(key); - data[key] = {std::move(in_data), std::move(in_shape)}; - map_iters[key] = lru_list.begin(); + // Insert new at front + lru_list_.emplace_front( + key, std::make_shared(std::move(data), std::move(shape))); + + map_[key] = lru_list_.begin(); + } + + void SetCapacity(size_t cap) { + std::lock_guard lock(mutex_); + capacity_ = cap; + + while (lru_list_.size() > capacity_) { + auto &last = lru_list_.back(); + size_t last_key = last.first; + + map_.erase(last_key); + lru_list_.pop_back(); + } + } + + size_t Size() const { + std::lock_guard lock(mutex_); + return lru_list_.size(); + } + + void Clear() { + std::lock_guard lock(mutex_); + map_.clear(); + lru_list_.clear(); } }; From 43a6f5f7f1d475d09f2b3af6469773bdc47598f5 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Fri, 20 Feb 2026 11:36:17 +1100 Subject: [PATCH 07/10] Update offline-tts-pocket-impl.h --- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 103 +++++++++------------ 1 file changed, 42 insertions(+), 61 deletions(-) diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 8db19f5593..0941d2b1b1 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -369,12 +371,21 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { return sentences; } - static size_t ComputeHash(const float *p, int32_t n) { - size_t hash = n; - hash ^= std::hash{}(n) + 0x9e3779b9 + (hash << 6) + (hash >> 2); - for (int32_t i = 0; i < n; ++i) { - hash ^= std::hash{}(p[i]) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + static size_t ComputeHash(const float *p, size_t n) { + size_t hash = 0; + + auto hash_combine = [](size_t &seed, size_t value) { + seed ^= value + 0x9e3779b97f4a7c15ull + (seed << 6) + (seed >> 2); + }; + + hash_combine(hash, n); + + for (size_t i = 0; i < n; ++i) { + uint32_t bits; + std::memcpy(&bits, &p[i], sizeof(float)); + hash_combine(hash, bits); } + return hash; } @@ -582,47 +593,10 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { num_samples = gen_config.reference_audio.size(); } - // Compute hash of reference audio for cache lookup - size_t audio_hash = ComputeHash(p_audio, num_samples); - - // Include max_reference_audio_len in hash to distinguish between same audio - // but different truncation settings. float max_reference_audio_len = gen_config.GetExtraFloat("max_reference_audio_len", 10); - audio_hash ^= std::hash{}(max_reference_audio_len) + 0x9e3779b9 + - (audio_hash << 6) + (audio_hash >> 2); - - // Check cache - // Check shared LRU cache - auto cached_embedding = cache_.Get(audio_hash); - if (cached_embedding) { - if (config_.model.debug) { - // Create an owned tensor and copy data to avoid use-after-free - Ort::AllocatorWithDefaultOptions allocator; - auto result = Ort::Value::CreateTensor( - allocator, cached_embedding->second.data(), - cached_embedding->second.size()); - std::copy(cached_embedding->first.begin(), - cached_embedding->first.end(), - result.GetTensorMutableData()); - - SHERPA_ONNX_LOGE("CACHE HIT: voice embedding (hash=%zu)", audio_hash); - return result; - } else { - // Create an owned tensor and copy data to avoid use-after-free - Ort::AllocatorWithDefaultOptions allocator; - auto result = Ort::Value::CreateTensor( - allocator, cached_embedding->second.data(), - cached_embedding->second.size()); - std::copy(cached_embedding->first.begin(), - cached_embedding->first.end(), - result.GetTensorMutableData()); - return result; - } - } // in seconds - // max_reference_audio_len is already computed above for the hash. int32_t max_len = static_cast(max_reference_audio_len * SampleRate()); @@ -638,6 +612,25 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { num_samples = max_len; } + // Compute hash of reference audio for cache lookup + size_t audio_hash = ComputeHash(p_audio, num_samples); + + // Check cache + // Check shared LRU cache + auto cached_embedding = cache_.Get(audio_hash); + if (cached_embedding) { + if (config_.model.debug) { + SHERPA_ONNX_LOGE("CACHE HIT: voice embedding (hash=%zu)", audio_hash); + } + // Create an owned tensor and copy data to avoid use-after-free + auto result = Ort::Value::CreateTensor( + model_->Allocator(), cached_embedding->second.data(), + cached_embedding->second.size()); + std::copy(cached_embedding->first.begin(), cached_embedding->first.end(), + result.GetTensorMutableData()); + return result; + } + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); @@ -648,29 +641,17 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { Ort::Value result = model_->RunMimiEncoder(std::move(x)); - if (config_.model.debug) { - // Cache the embedding in shared LRU cache - auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); - size_t total = std::accumulate(result_shape.begin(), result_shape.end(), - 1, std::multiplies()); - const float *result_data = result.GetTensorData(); + auto info = result.GetTensorTypeAndShapeInfo(); + auto result_shape = info.GetShape(); + size_t total = info.GetElementCount(); + const float *result_data = result.GetTensorData(); - cache_.Put( - audio_hash, std::vector(result_data, result_data + total), - std::vector(result_shape.begin(), result_shape.end())); + cache_.Put(audio_hash, std::vector(result_data, result_data + total), + std::move(result_shape)); + if (config_.model.debug) { SHERPA_ONNX_LOGE("CACHE MISS: cached embedding (hash=%zu, %zu floats)", audio_hash, total); - } else { - // Cache the embedding in shared LRU cache (no logging) - auto result_shape = result.GetTensorTypeAndShapeInfo().GetShape(); - size_t total = std::accumulate(result_shape.begin(), result_shape.end(), - 1, std::multiplies()); - const float *result_data = result.GetTensorData(); - - cache_.Put( - audio_hash, std::vector(result_data, result_data + total), - std::vector(result_shape.begin(), result_shape.end())); } return result; From 5405b1123f5c11853d45addee052a4bd82642c34 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Fri, 20 Feb 2026 13:55:01 +1100 Subject: [PATCH 08/10] Update sherpa-onnx/csrc/offline-tts-pocket-impl.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 0941d2b1b1..79572b0484 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -615,8 +615,6 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { // Compute hash of reference audio for cache lookup size_t audio_hash = ComputeHash(p_audio, num_samples); - // Check cache - // Check shared LRU cache auto cached_embedding = cache_.Get(audio_hash); if (cached_embedding) { if (config_.model.debug) { From 9a61ad86219f0c0a2f4f01adc22bf91ece882c1f Mon Sep 17 00:00:00 2001 From: Simon Law Date: Fri, 20 Feb 2026 13:55:18 +1100 Subject: [PATCH 09/10] Update sherpa-onnx/csrc/offline-tts-pocket-impl.h Co-authored-by: Fangjun Kuang --- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 79572b0484..7dee493e40 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -830,7 +830,7 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { map_[key] = lru_list_.begin(); } - void SetCapacity(size_t cap) { + void SetCapacity(int32_t cap) { std::lock_guard lock(mutex_); capacity_ = cap; From b7e6124986d54bc4515a721ca58f4bdac2285583 Mon Sep 17 00:00:00 2001 From: Simon Law Date: Fri, 20 Feb 2026 20:55:31 +1100 Subject: [PATCH 10/10] check check cap is valid --- sherpa-onnx/csrc/offline-tts-pocket-impl.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sherpa-onnx/csrc/offline-tts-pocket-impl.h b/sherpa-onnx/csrc/offline-tts-pocket-impl.h index 7dee493e40..7e8284454c 100644 --- a/sherpa-onnx/csrc/offline-tts-pocket-impl.h +++ b/sherpa-onnx/csrc/offline-tts-pocket-impl.h @@ -831,6 +831,12 @@ class OfflineTtsPocketImpl : public OfflineTtsImpl { } void SetCapacity(int32_t cap) { + if (cap < 0) { + SHERPA_ONNX_LOGE( + "voice_embedding_cache_capacity must be >= 0. Given: %d", cap); + SHERPA_ONNX_EXIT(-1); + } + std::lock_guard lock(mutex_); capacity_ = cap;