feat: add LFM2-Audio / LFM2.5-Audio speech input and output support

.gitignore

@@ -30,6 +30,8 @@ examples/python/hf_cache
 examples/csharp/ModelChat/models
 
 !test/models/gemma4/*
+!/test/models/lfm2-audio/
+!test/models/lfm2-audio/*
 !test/models/hf-internal-testing/*
 !test/models/lfm2-vl/*
 !test/models/marian-batch/*

README.md

@@ -16,7 +16,7 @@ See documentation at the [ONNX Runtime website](https://onnxruntime.ai/docs/gena
 
 | Support matrix | Supported now | Under development | On the roadmap|
 | -------------- | ------------- | ----------------- | -------------- |
-| Model architectures | AMD OLMo <br/> ChatGLM <br/> DeepSeek <br/> ERNIE 4.5 <br/> Fara <br/> Gemma <br/> gpt-oss <br/> Granite <br/> Granite MoE Hybrid <br/> HunYuan Dense V1 <br/> InternLM2 <br/> LFM2 (language + vision) <br/> Llama <br/> Mistral <br/> Nemotron <br/> Phi (language + vision) <br/> Qwen (language + vision) <br/> SmolLM3 <br/> Whisper | Stable diffusion | Multi-modal models |
+| Model architectures | AMD OLMo <br/> ChatGLM <br/> DeepSeek <br/> ERNIE 4.5 <br/> Fara <br/> Gemma <br/> gpt-oss <br/> Granite <br/> Granite MoE Hybrid <br/> HunYuan Dense V1 <br/> InternLM2 <br/> LFM2 (language + vision + speech input) <br/> Llama <br/> Mistral <br/> Nemotron <br/> Phi (language + vision) <br/> Qwen (language + vision) <br/> SmolLM3 <br/> Whisper | Stable diffusion | Multi-modal models |
 | API | Python <br/>C# <br/>C/C++ <br/> Java ^ | Objective-C ||
 | O/S | Linux <br/> Windows <br/>Mac  <br/>Android || iOS |||
 | Architecture | x86 <br/> x64 <br/> arm64 ||||

examples/python/common.py

@@ -322,6 +322,11 @@ def get_user_content(model_type: str, num_images: int, num_audios: int, prompt:
         # per projected vision feature, and <|image_end|>, based on the resized image's patch grid.
         image_tags = "".join(["<image>" for _ in range(num_images)])
         content = image_tags + prompt
+    elif model_type == "lfm2_audio":
+        # LFM2-Audio: the C++ audio processor replaces each <|audio|> with one placeholder per
+        # encoder frame (one per 80 ms of audio) and splices the encoder output in at those positions.
+        audio_tags = "".join(["<|audio|>" for _ in range(num_audios)])
+        content = audio_tags + prompt
     elif model_type == "mistral3":
         # Pixtral / Ministral-3 VLM: the C++ image processor expands each
         # [IMG] into the full token sequence based on image resolution.

src/config.cpp

@@ -1473,6 +1473,8 @@ struct SpeechInputs_Element : JSON::Element {
       v_.attention_mask = JSON::Get<std::string_view>(value);
     } else if (name == "audio_sizes") {
       v_.audio_sizes = JSON::Get<std::string_view>(value);
+    } else if (name == "audio_lengths") {
+      v_.audio_lengths = JSON::Get<std::string_view>(value);
     } else if (name == "audio_projection_mode") {
       v_.audio_projection_mode = JSON::Get<std::string_view>(value);
     } else {
@@ -1542,6 +1544,68 @@ struct Speech_Element : JSON::Element {
   SpeechOutputs_Element outputs_{v_.outputs};
 };
 
+struct AudioOutputGraph_Element : JSON::Element {
+  explicit AudioOutputGraph_Element(Config::Model::AudioOutput::Graph& v) : v_{v} {}
+
+  void OnValue(std::string_view name, JSON::Value value) override {
+    if (name == "filename") {
+      v_.filename = JSON::Get<std::string_view>(value);
+    } else {
+      throw JSON::unknown_value_error{};
+    }
+  }
+
+  Element& OnObject(std::string_view name) override {
+    if (name == "session_options") {
+      v_.session_options = Config::SessionOptions{};
+      session_options_ = std::make_unique<SessionOptions_Element>(*v_.session_options);
+      return *session_options_;
+    }
+    throw JSON::unknown_value_error{};
+  }
+
+ private:
+  Config::Model::AudioOutput::Graph& v_;
+  std::unique_ptr<SessionOptions_Element> session_options_;
+};
+
+struct AudioOutput_Element : JSON::Element {
+  explicit AudioOutput_Element(Config::Model::AudioOutput& v) : v_{v} {}
+
+  void OnValue(std::string_view name, JSON::Value value) override {
+    if (name == "num_codebooks") {
+      v_.num_codebooks = SafeDoubleToInt(JSON::Get<double>(value), name);
+    } else if (name == "codebook_size") {
+      v_.codebook_size = SafeDoubleToInt(JSON::Get<double>(value), name);
+    } else if (name == "audio_start_token_id") {
+      v_.audio_start_token_id = SafeDoubleToInt(JSON::Get<double>(value), name);
+    } else if (name == "text_end_token_id") {
+      v_.text_end_token_id = SafeDoubleToInt(JSON::Get<double>(value), name);
+    } else if (name == "interleaved_n_text") {
+      v_.interleaved_n_text = SafeDoubleToInt(JSON::Get<double>(value), name);
+    } else if (name == "interleaved_n_audio") {
+      v_.interleaved_n_audio = SafeDoubleToInt(JSON::Get<double>(value), name);
+    } else {
+      throw JSON::unknown_value_error{};
+    }
+  }
+
+  Element& OnObject(std::string_view name) override {
+    if (name == "depthformer") {
+      return depthformer_;
+    }
+    if (name == "embedding") {
+      return embedding_;
+    }
+    throw JSON::unknown_value_error{};
+  }
+
+ private:
+  Config::Model::AudioOutput& v_;
+  AudioOutputGraph_Element depthformer_{v_.depthformer};
+  AudioOutputGraph_Element embedding_{v_.embedding};
+};
+
 struct JoinerInputs_Element : JSON::Element {
   explicit JoinerInputs_Element(Config::Model::Joiner::Inputs& v) : v_{v} {}
 
@@ -2160,6 +2224,9 @@ struct Model_Element : JSON::Element {
     if (name == "speech") {
       return speech_;
     }
+    if (name == "audio_output") {
+      return audio_output_;
+    }
     if (name == "joiner") {
       return joiner_;
     }
@@ -2196,6 +2263,7 @@ struct Model_Element : JSON::Element {
   Vision_Element vision_{v_.vision};
   Embedding_Element embedding_{v_.embedding};
   Speech_Element speech_{v_.speech};
+  AudioOutput_Element audio_output_{v_.audio_output};
   Joiner_Element joiner_{v_.joiner};
   VAD_Element vad_{v_.vad};
   Moonshine_Element moonshine_{v_.moonshine};
@@ -2310,6 +2378,12 @@ struct Search_Element : JSON::Element {
       v_.early_stopping = JSON::Get<bool>(value);
     } else if (name == "blank_penalty") {
       v_.blank_penalty = static_cast<float>(JSON::Get<double>(value));
+    } else if (name == "audio_interleaved") {
+      v_.audio_interleaved = JSON::Get<bool>(value);
+    } else if (name == "audio_temperature") {
+      v_.audio_temperature = static_cast<float>(JSON::Get<double>(value));
+    } else if (name == "audio_top_k") {
+      v_.audio_top_k = SafeDoubleToInt(JSON::Get<double>(value), name);
     } else {
       throw JSON::unknown_value_error{};
     }

src/config.h

@@ -48,6 +48,7 @@ struct Config {
     // Speech encoder names
     static constexpr std::string_view AudioAttentionMaskName = "audio_attention_mask";
     static constexpr std::string_view AudioSizesName = "audio_sizes";
+    static constexpr std::string_view AudioLengthsName = "audio_lengths";
     static constexpr std::string_view AudioProjectionModeName = "audio_projection_mode";
     static constexpr std::string_view AudioFeaturesName = "audio_features";
     static constexpr std::string_view NumAudioTokens = "num_audio_tokens";
@@ -317,6 +318,7 @@ struct Config {
         std::string audio_embeds{Defaults::AudioEmbedsName};
         std::string attention_mask{Defaults::AudioAttentionMaskName};
         std::string audio_sizes{Defaults::AudioSizesName};
+        std::string audio_lengths{Defaults::AudioLengthsName};  // per-clip valid frame count of audio_embeds (LFM2-Audio)
         std::string audio_projection_mode{Defaults::AudioProjectionModeName};
       } inputs;
 
@@ -325,6 +327,24 @@ struct Config {
       } outputs;
     } speech;
 
+    // Speech output of LFM2-Audio: the depthformer turns one decoder hidden state into a frame of
+    // audio codes, one per codebook, and the audio embedding turns that frame back into the decoder's
+    // next input. Empty filenames leave the model answering in text only.
+    struct AudioOutput {
+      struct Graph {
+        std::string filename;
+        std::optional<SessionOptions> session_options;
+      };
+      Graph depthformer;
+      Graph embedding;
+      int num_codebooks{8};
+      int codebook_size{2049};        // entries per codebook; the last one is the end-of-audio code
+      int audio_start_token_id{128};  // <|audio_start|>: the rest of a sequential answer is speech
+      int text_end_token_id{130};     // <|text_end|>: the text of an interleaved answer is over
+      int interleaved_n_text{6};      // text tokens per turn of an interleaved answer
+      int interleaved_n_audio{12};    // audio frames per turn of an interleaved answer
+    } audio_output;
+
     struct Joiner {
       std::string filename;
       std::optional<SessionOptions> session_options;
@@ -734,6 +754,9 @@ struct Config {
     int random_seed{-1};               // -1 = Seed with random device, otherwise use value to seed RNG
     std::optional<size_t> chunk_size;  // Chunk size for prefill chunking during context processing. If present, chunking is enabled with the chunk size > 0.
     float blank_penalty{};             // Penalty applied to blank token logits in CTC/RNNT decoding. Default 0 means no penalty.
+    bool audio_interleaved{};          // LFM2-Audio: alternate text tokens and audio frames by count (interleaved mode) rather than switching on <|audio_start|>.
+    float audio_temperature{1.0f};     // LFM2-Audio: temperature the audio codes are sampled with. 0 takes the most likely code.
+    int audio_top_k{4};                // LFM2-Audio: number of most likely audio codes kept when sampling. 1 takes the most likely code.
   } search;
 
   struct Speculative {

src/generator/generators.cpp

@@ -971,7 +971,7 @@ void Generator::RewindToLength(size_t new_length) {
   if (ModelType::IsTransducer(model_type) || ModelType::IsStreamingEncDecASR(model_type))
     throw std::runtime_error("RewindTo is not supported for streaming ASR models (" + model_type + ").");
   if (model_type == "whisper" || model_type == "phi3v" || model_type == "decoder-pipeline" ||
-      ModelType::IsLFM2(model_type) || model_type == "lfm2_vl")
+      ModelType::IsLFM2(model_type) || model_type == "lfm2_vl" || model_type == "lfm2_audio")
     throw std::runtime_error("RewindTo is currently not supported for " + model_type + ".");
   const size_t current_length = search_->GetSequenceLength();
   if (new_length > current_length)

src/generator/generators.h

@@ -70,6 +70,12 @@ DeviceSpan<T> WrapTensor(DeviceInterface& device, OrtValue& value) {
 
 DeviceSpan<uint8_t> ByteWrapTensor(DeviceInterface& device, OrtValue& value);
 
+// The interface that matches where a tensor's memory actually is. A model that runs on a device does
+// not necessarily keep its tensors there: WebGPU without graph capture leaves them host-side, and ORT
+// places some outputs on the CPU whatever the EP. Wrapping such a tensor with the device interface
+// hands a host pointer to the EP's copy, which crashes inside it.
+DeviceInterface& DeviceForTensor(const OrtValue& value, DeviceInterface& device);
+
 // OgaSequences are a vector of int32 vectors
 using TokenSequences = std::vector<std::vector<int32_t>>;