Add Qwen VL/3.5 support and refactor model builder infrastructure

src/config.cpp

@@ -353,6 +353,8 @@ struct DecoderInputs_Element : JSON::Element {
       v_.attention_metadata = JSON::Get<std::string_view>(value);
     } else if (name == "past_conv_names") {
       v_.past_conv_names = JSON::Get<std::string_view>(value);
+    } else if (name == "past_recurrent_names") {
+      v_.past_recurrent_names = JSON::Get<std::string_view>(value);
     } else if (name == "targets") {
       v_.targets = JSON::Get<std::string_view>(value);
     } else if (name == "lstm_hidden_state") {
@@ -390,6 +392,8 @@ struct DecoderOutputs_Element : JSON::Element {
       v_.rnn_states = JSON::Get<std::string_view>(value);
     } else if (name == "present_conv_names") {
       v_.present_conv_names = JSON::Get<std::string_view>(value);
+    } else if (name == "present_recurrent_names") {
+      v_.present_recurrent_names = JSON::Get<std::string_view>(value);
     } else if (name == "outputs") {
       v_.outputs = JSON::Get<std::string_view>(value);
     } else if (name == "lstm_hidden_state") {

src/config.h

@@ -23,9 +23,13 @@ struct Config {
     static constexpr std::string_view PositionIdsName = "position_ids";
     static constexpr std::string_view PastKeyName = "past_key_values.%d.key";
     static constexpr std::string_view PastValueName = "past_key_values.%d.value";
+    static constexpr std::string_view PastConvName = "past.%d.conv";
+    static constexpr std::string_view PastRecurrentName = "past.%d.recurrent";
     static constexpr std::string_view LogitsName = "logits";
     static constexpr std::string_view PresentKeyName = "present.%d.key";
     static constexpr std::string_view PresentValueName = "present.%d.value";
+    static constexpr std::string_view PresentConvName = "present.%d.conv";
+    static constexpr std::string_view PresentRecurrentName = "present.%d.recurrent";
     static constexpr std::string_view RnnStatesName = "rnn_states";
     static constexpr std::string_view RnnStatesPrevName = "rnn_states_prev";
     static constexpr std::string_view CumulativeSequenceLengthsName = "cumulative_sequence_lengths";
@@ -40,6 +44,7 @@ struct Config {
     static constexpr std::string_view AudioProjectionModeName = "audio_projection_mode";
     static constexpr std::string_view AudioFeaturesName = "audio_features";
     static constexpr std::string_view NumAudioTokens = "num_audio_tokens";
+    static constexpr std::string_view OutputCrossQKName = "output_cross_qk_%d";
 
     // Vision encoder names
     static constexpr std::string_view PixelValuesName = "pixel_values";
@@ -385,7 +390,8 @@ struct Config {
         std::string past_sequence_lengths{Defaults::PastSequenceLengthsName};
         std::string block_table{Defaults::BlockTableName};
         std::string attention_metadata{Defaults::AttentionMetadataName};
-        std::string past_conv_names{"past_conv.%d"};  // Conv cache input name template (LFM2)
+        std::string past_conv_names{Defaults::PastConvName};  // Conv cache input name template (LFM2)
+        std::string past_recurrent_names{Defaults::PastRecurrentName};
 
         // RNNT decoder inputs
         std::string targets;
@@ -404,9 +410,10 @@ struct Config {
         std::string present_key_names{Defaults::PresentKeyName};
         std::string present_value_names{Defaults::PresentValueName};
         std::string present_names;  // When key/value pairs are combined
-        std::string output_cross_qk_names{"output_cross_qk_%d"};
+        std::string output_cross_qk_names{Defaults::OutputCrossQKName};
         std::string rnn_states{Defaults::RnnStatesName};
-        std::string present_conv_names{"present_conv.%d"};  // Conv cache output name template (LFM2)
+        std::string present_conv_names{Defaults::PresentConvName};  // Conv cache output name template (LFM2)
+        std::string present_recurrent_names{Defaults::PresentRecurrentName};
 
         // RNNT decoder outputs
         std::string outputs;

src/models/model_type.h

@@ -17,7 +17,7 @@ namespace Generators {
 struct ModelType {
   inline static bool IsLLM(const std::string& model_type) {
     // Large-language model (LLM)
-    static constexpr std::array<std::string_view, 27> LLM = {"chatglm", "decoder", "ernie4_5", "gemma", "gemma2", "gemma3_text", "gemma4_text", "gpt2", "gptoss", "granite", "granitemoehybrid", "hunyuandensev1", "internlm2", "lfm2", "llama", "mistral", "nemotron", "olmo", "phi", "phimoe", "phi3", "phi3small", "qwen2", "qwen3", "qwen3_5_moe_text", "qwen3_5_text", "smollm3"};
+    static constexpr std::array<std::string_view, 31> LLM = {"chatglm", "decoder", "ernie4_5", "gemma", "gemma2", "gemma3_text", "gemma3_vl_text", "gemma4_text", "gpt2", "gptoss", "granite", "granitemoehybrid", "hunyuandensev1", "internlm2", "lfm2", "llama", "mistral", "mistral3_text", "nemotron", "olmo", "phi", "phimoe", "phi3", "phi3small", "qwen2", "qwen2_5_vl_text", "qwen3", "qwen3_vl_text", "qwen3_5_moe_text", "qwen3_5_text", "smollm3"};
     return std::find(LLM.begin(), LLM.end(), model_type) != LLM.end();
   }
 
@@ -27,23 +27,24 @@ struct ModelType {
     return std::find(VLM.begin(), VLM.end(), model_type) != VLM.end();
   }
 
-  inline static bool IsQwenVLFamily(const std::string& model_type) {
-    // Qwen-VL family: models requiring 3D mRoPE position IDs
-    static constexpr std::array<std::string_view, 5> QwenVL = {"fara", "qwen2_5_vl", "qwen3_vl", "qwen3_5", "qwen3_5_moe"};
-    return std::find(QwenVL.begin(), QwenVL.end(), model_type) != QwenVL.end();
-  }
-
-  inline static bool IsPixtralFamily(const std::string& model_type) {
-    // Pixtral family: per-image vision loop with variable resolution
-    return model_type == "mistral3";
-  }
-
   inline static bool IsALM(const std::string& model_type) {
     // Audio-language model (ALM)
     static constexpr std::array<std::string_view, 1> ALM = {"whisper"};
     return std::find(ALM.begin(), ALM.end(), model_type) != ALM.end();
   }
 
+  inline static bool IsMMM(const std::string& model_type) {
+    // Multi-modal model (MMM)
+    static constexpr std::array<std::string_view, 2> MMM = {"gemma4", "phi4mm"};
+    return std::find(MMM.begin(), MMM.end(), model_type) != MMM.end();
+  }
+
+  inline static bool IsPipe(const std::string& model_type) {
+    // Pipeline (Pipe)
+    static constexpr std::array<std::string_view, 1> Pipe = {"decoder-pipeline"};
+    return std::find(Pipe.begin(), Pipe.end(), model_type) != Pipe.end();
+  }
+
   inline static bool IsRNNT(const std::string& model_type) {
     // RNNT models bypass the search/logits pipeline entirely.
     static constexpr std::array<std::string_view, 1> rnnt_types = {"nemotron_speech"};
@@ -61,16 +62,15 @@ struct ModelType {
     return IsRNNT(model_type) || IsTDT(model_type);
   }
 
-  inline static bool IsMMM(const std::string& model_type) {
-    // Multi-modal model (MMM)
-    static constexpr std::array<std::string_view, 2> MMM = {"gemma4", "phi4mm"};
-    return std::find(MMM.begin(), MMM.end(), model_type) != MMM.end();
+  inline static bool IsQwenVLFamily(const std::string& model_type) {
+    // Qwen-VL family: models requiring 3D mRoPE position IDs
+    static constexpr std::array<std::string_view, 10> QwenVL = {"fara", "qwen2_5_vl_text", "qwen2_5_vl", "qwen3_vl", "qwen3_vl_text", "qwen3_5", "qwen3_5_moe", "qwen3_5_moe_text", "qwen3_5_text", "videochat_flash_qwen"};
+    return std::find(QwenVL.begin(), QwenVL.end(), model_type) != QwenVL.end();
   }
 
-  inline static bool IsPipe(const std::string& model_type) {
-    // Pipeline (Pipe)
-    static constexpr std::array<std::string_view, 1> Pipe = {"decoder-pipeline"};
-    return std::find(Pipe.begin(), Pipe.end(), model_type) != Pipe.end();
+  inline static bool IsPixtralFamily(const std::string& model_type) {
+    // Pixtral family: per-image vision loop with variable resolution
+    return model_type == "mistral3";
   }
 
   inline static bool IsLFM2(const std::string& model_type) {

src/models/recurrent_state.cpp

@@ -9,31 +9,30 @@
 
 namespace Generators {
 
+std::string ComposeCacheName(const std::string& template_string, int index) {
+  constexpr int32_t CacheNameLength = 64;
+  char cache_name[CacheNameLength];
+  if (auto length = snprintf(cache_name, std::size(cache_name), template_string.c_str(), index);
+      length < 0 || length >= CacheNameLength) {
+    throw std::runtime_error("Unable to compose cache name from the provided template " + template_string +
+                             ". This could be either due to an encoding error or the name being too long.");
+  }
+  return std::string(cache_name);
+}
+
 RecurrentState::RecurrentState(State& state)
     : state_{state} {
-  const auto& past_key_template = model_.config_->model.decoder.inputs.past_key_names;
-  const auto& present_key_template = model_.config_->model.decoder.outputs.present_key_names;
-
-  // Derive recurrent name templates from KV name templates
-  auto derive_template = [](const std::string& kv_template, const std::string& suffix) -> std::string {
-    auto pos = kv_template.rfind('.');
-    if (pos == std::string::npos) return "";
-    return kv_template.substr(0, pos + 1) + suffix;
-  };
-
-  std::string past_conv_template = derive_template(past_key_template, "conv_state");
-  std::string past_recurrent_template = derive_template(past_key_template, "recurrent_state");
-  std::string present_conv_template = derive_template(present_key_template, "conv_state");
-  std::string present_recurrent_template = derive_template(present_key_template, "recurrent_state");
+  // Discover recurrent layer indices by scanning all session input names
+  const auto& past_recurrent_template = model_.config_->model.decoder.inputs.past_recurrent_names;
+  const auto placeholder_pos = past_recurrent_template.find("%d");
+  if (placeholder_pos == std::string::npos) return;
 
-  if (past_conv_template.empty()) return;
+  const auto prefix = past_recurrent_template.substr(0, placeholder_pos);
+  const auto suffix = past_recurrent_template.substr(placeholder_pos + 2);
 
-  // Discover recurrent layer indices by scanning all session input names
   for (const auto& name : model_.session_info_.GetInputNames()) {
-    // Try to match against the conv_state template (e.g. "past_key_values.%d.conv_state")
+    // Try to match against the recurrent state template (e.g. "past.%d.recurrent")
     // Extract the layer index from names that match
-    auto prefix = past_conv_template.substr(0, past_conv_template.find('%'));
-    auto suffix = past_conv_template.substr(past_conv_template.find('%') + 2);  // skip %d
     if (name.size() > prefix.size() + suffix.size() &&
         name.compare(0, prefix.size(), prefix) == 0 &&
         name.compare(name.size() - suffix.size(), suffix.size(), suffix) == 0) {
@@ -56,10 +55,10 @@ RecurrentState::RecurrentState(State& state)
                       return s; }() + ")");
 
   for (int idx : layer_indices_) {
-    input_name_strings_.push_back(ComposeKeyValueName(past_conv_template, idx));
-    input_name_strings_.push_back(ComposeKeyValueName(past_recurrent_template, idx));
-    output_name_strings_.push_back(ComposeKeyValueName(present_conv_template, idx));
-    output_name_strings_.push_back(ComposeKeyValueName(present_recurrent_template, idx));
+    input_name_strings_.push_back(ComposeCacheName(model_.config_->model.decoder.inputs.past_conv_names, idx));
+    input_name_strings_.push_back(ComposeCacheName(model_.config_->model.decoder.inputs.past_recurrent_names, idx));
+    output_name_strings_.push_back(ComposeCacheName(model_.config_->model.decoder.outputs.present_conv_names, idx));
+    output_name_strings_.push_back(ComposeCacheName(model_.config_->model.decoder.outputs.present_recurrent_names, idx));
   }
 
   conv_type_ = model_.session_info_.GetInputDataType(input_name_strings_[0]);
@@ -83,8 +82,8 @@ RecurrentState::RecurrentState(State& state)
                                  std::to_string(shape[i]) + " at axis " + std::to_string(i));
     }
   };
-  validate_shape(conv_shape_, "conv_state");
-  validate_shape(recurrent_shape_, "recurrent_state");
+  validate_shape(conv_shape_, "conv");
+  validate_shape(recurrent_shape_, "recurrent");
 
   const int num_layers = static_cast<int>(layer_indices_.size());
 

src/python/py/models/README.md

@@ -585,7 +585,7 @@ The scale file is produced by the `kv_cache_calibration` module, which runs a ba
 python -m onnxruntime_genai.models.builder -i path_to_local_folder_on_disk -o path_to_baseline_folder -p precision -e cuda -c cache_dir_to_store_temp_files
 
 # 2. Calibrate the scales:
-python -m onnxruntime_genai.models.kv_cache_calibration --model path_to_baseline_folder --tokenizer path_to_local_folder_on_disk --out path_to_scales.json --quant-type int8_per_channel
+python -m onnxruntime_genai.models.quantization.kv_cache_calibration --model path_to_baseline_folder --tokenizer path_to_local_folder_on_disk --out path_to_scales.json --quant-type int8_per_channel
 ```
 
 Then rebuild with the quantized KV cache: