diff --git a/src/config.cpp b/src/config.cpp index 9fc7f3605b..db8be5907b 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// Modifications Copyright(C) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #include "generators.h" #include "models/model_type.h" #include "runtime_settings.h" @@ -778,6 +779,8 @@ struct Vision_Element : JSON::Element { v_.tokens_per_second = static_cast(JSON::Get(value)); } else if (name == "patch_size") { v_.patch_size = static_cast(JSON::Get(value)); + } else if (name == "num_visual_tokens") { + v_.num_visual_tokens = static_cast(JSON::Get(value)); } else if (name == "window_size") { v_.window_size = static_cast(JSON::Get(value)); } else { diff --git a/src/config.h b/src/config.h index 155970ba22..e2e4f6e869 100644 --- a/src/config.h +++ b/src/config.h @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// Modifications Copyright(C) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #pragma once namespace Generators { @@ -218,10 +219,11 @@ struct Config { // and these values are unused. int spatial_merge_size{2}; float tokens_per_second{2.0f}; - int patch_size{14}; // Qwen2.5-VL uses 14, Qwen3-VL uses 16 - int window_size{0}; // Used by CalculateWindowIndex() in QNN pipeline only. - // 0 = auto-compute as patch_size * spatial_merge_size * 2 - // Qwen2.5-VL default: 56 (14*4), Qwen3-VL default: 64 (16*4) + int num_visual_tokens{0}; // Fixed visual tokens per image; must be > 0 for videochat_flash_qwen + int patch_size{14}; // Qwen2.5-VL uses 14, Qwen3-VL uses 16 + int window_size{0}; // Used by CalculateWindowIndex() in QNN pipeline only. + // 0 = auto-compute as patch_size * spatial_merge_size * 2 + // Qwen2.5-VL default: 56 (14*4), Qwen3-VL default: 64 (16*4) std::string config_filename{"processor_config.json"}; std::optional adapter_filename{}; diff --git a/src/models/model.cpp b/src/models/model.cpp index 8a4c248cf5..f93c697f65 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // -// Modifications Copyright(C) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #include #include #include @@ -23,6 +24,7 @@ #include "decoder_only_pipeline.h" #include "qwen_vl_model.h" #include "qwen2_5_vl_image_processor.h" +#include "videochat_flash_processor.h" #include "mistral3_image_processor.h" #include "../dml/interface.h" #include "../openvino/interface.h" @@ -943,7 +945,8 @@ MultiModalProcessor::MultiModalProcessor(Config& config, const SessionInfo& sess {"fara", Processor::Create}, {"qwen2_5_vl", Processor::Create}, {"qwen3_vl", Processor::Create}, - {"qwen3_5", Processor::Create}} { + {"qwen3_5", Processor::Create}, + {"videochat_flash_qwen", Processor::Create}} { auto processor = processor_factory_.find(config.model.type); if (processor != processor_factory_.end()) { processor_ = processor->second(config, session_info); diff --git a/src/models/model.h b/src/models/model.h index cd47f37112..6a52fe6934 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // -// Modifications Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #pragma once #include "model_type.h" #include "ortx_tokenizer.h" diff --git a/src/models/model_type.h b/src/models/model_type.h index e34dd348f0..24bd7f6e50 100644 --- a/src/models/model_type.h +++ b/src/models/model_type.h @@ -21,7 +21,7 @@ struct ModelType { inline static bool IsVLM(const std::string& model_type) { // Vision-language model (VLM) - static constexpr std::array VLM = {"fara", "gemma3", "mistral3", "phi3v", "qwen2_5_vl", "qwen3_vl", "qwen3_5"}; + static constexpr std::array VLM = {"fara", "gemma3", "mistral3", "phi3v", "qwen2_5_vl", "qwen3_vl", "qwen3_5", "videochat_flash_qwen"}; return std::find(VLM.begin(), VLM.end(), model_type) != VLM.end(); } diff --git a/src/models/videochat_flash_processor.cpp b/src/models/videochat_flash_processor.cpp new file mode 100644 index 0000000000..2f2c57c108 --- /dev/null +++ b/src/models/videochat_flash_processor.cpp @@ -0,0 +1,225 @@ +// Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. +// +// SPDX-License-Identifier: MIT +// +// Licensed under the MIT License. See License.txt in the project root for +// license information. + +#include "../generators.h" +#include "model.h" +#include "videochat_flash_processor.h" +#include + +namespace Generators { + +namespace { + +// Build input_ids from prompt, inserting fixed_tokens_per_image <|image_pad|> tokens per image. +std::tuple, std::unique_ptr> +BuildPromptTokens(const Tokenizer& tokenizer, const std::string& prompt, + int64_t num_images, int64_t tokens_per_image, + Ort::Allocator& allocator) { + constexpr char vision_start_token[] = "<|vision_start|>"; + constexpr char vision_end_token[] = "<|vision_end|>"; + constexpr char image_pad_token[] = "<|image_pad|>"; + + std::string text = prompt; + int64_t total_image_tokens = num_images * tokens_per_image; + + // Verify prompt has the right number of vision_start markers + const std::regex vision_start_regex{R"(<\|vision_start\|>)"}; + auto begin = std::sregex_iterator(text.begin(), text.end(), vision_start_regex); + auto end = std::sregex_iterator(); + int64_t marker_count = std::distance(begin, end); + + if (num_images > 0 && marker_count != num_images) { + throw std::runtime_error("Prompt contained " + std::to_string(marker_count) + + " vision_start tokens but received " + std::to_string(num_images) + " images."); + } + + // Replace each <|vision_start|>...<|vision_end|> block with the correct pad count + if (num_images > 0) { + std::string modified; + size_t last_pos = 0; + std::string temp = text; + std::smatch match; + + while (std::regex_search(temp, match, vision_start_regex)) { + size_t abs_pos = match.position() + (text.size() - temp.size()); + modified += text.substr(last_pos, abs_pos - last_pos); + + modified += vision_start_token; + for (int64_t i = 0; i < tokens_per_image; ++i) + modified += image_pad_token; + modified += vision_end_token; + + last_pos = abs_pos + match.length(); + size_t ve_pos = text.find(vision_end_token, last_pos); + if (ve_pos != std::string::npos) + last_pos = ve_pos + strlen(vision_end_token); + + temp = match.suffix(); + } + modified += text.substr(last_pos); + text = modified; + } + + const std::vector input_ids = tokenizer.Encode(text.c_str()); + + auto input_ids_value = OrtValue::CreateTensor( + allocator, std::vector{1, static_cast(input_ids.size())}); + std::copy(input_ids.begin(), input_ids.end(), input_ids_value->GetTensorMutableData()); + + auto num_img_tokens = OrtValue::CreateTensor(allocator, std::vector{1}); + num_img_tokens->GetTensorMutableData()[0] = total_image_tokens; + + return {std::move(input_ids_value), std::move(num_img_tokens)}; +} + +} // namespace + +VideoChatFlashProcessor::VideoChatFlashProcessor(Config& config, const SessionInfo& session_info) + : pixel_values_type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT}, + num_visual_tokens_{config.model.vision.num_visual_tokens} { + if (num_visual_tokens_ <= 0) + throw std::runtime_error("videochat_flash_qwen requires vision.num_visual_tokens > 0 in genai_config.json"); + + const auto processor_config = (config.config_path / fs::path(config.model.vision.config_filename)).string(); + CheckResult(OrtxCreateProcessor(processor_.ToBeAssigned(), processor_config.c_str())); + + try { + pixel_values_type_ = session_info.GetInputDataType(config.model.vision.inputs.pixel_values); + } catch (...) { + // pixel_values input may be absent when only the language decoder session is loaded; + // the default-initialized pixel_values_type_ (FLOAT) is used in that case. + } + + config.AddMapping(std::string(Config::Defaults::InputIdsName), config.model.embedding.inputs.input_ids); + config.AddMapping(std::string(Config::Defaults::PixelValuesName), config.model.vision.inputs.pixel_values); +} + +std::unique_ptr VideoChatFlashProcessor::Process(const Tokenizer& tokenizer, const Payload& payload) const { + std::string prompt = std::string(payload.prompt); + const Images* images = payload.images; + Ort::Allocator& allocator{Ort::Allocator::GetWithDefaultOptions()}; + auto named_tensors = std::make_unique(); + + // Text-only: no image processing needed + if (!images || images->num_images_ == 0) { + auto [input_ids, num_img_tokens] = BuildPromptTokens(tokenizer, prompt, 0, 0, allocator); + named_tensors->emplace(std::string(Config::Defaults::InputIdsName), + std::make_shared(std::move(input_ids))); + named_tensors->emplace(std::string(Config::Defaults::NumImageTokens), + std::make_shared(std::move(num_img_tokens))); + return named_tensors; + } + + // Run ORT Extensions image preprocessing (Decode → Resize → Rescale → Normalize) + ort_extensions::OrtxObjectPtr result; + CheckResult(OrtxImagePreProcess(processor_.get(), images->images_.get(), result.ToBeAssigned())); + + ort_extensions::OrtxObjectPtr pixel_values_owner; + CheckResult(OrtxTensorResultGetAt(result.get(), 0, pixel_values_owner.ToBeAssigned())); + OrtxTensor* pixel_values = pixel_values_owner.get(); + + const float* pv_data{}; + const int64_t* pv_shape{}; + size_t pv_ndims; + CheckResult(OrtxGetTensorData(pixel_values, reinterpret_cast(&pv_data), + &pv_shape, &pv_ndims)); + + // Detect whether ORT Extensions output is HWC or CHW. + // Once processor_config.json includes a Permute3D step, the output will be + // NCHW and the HWC path below can be removed. + int64_t num_imgs, channels, height, width; + bool is_hwc; + if (pv_ndims == 3) { + num_imgs = 1; + // CHW: [C, H, W] vs HWC: [H, W, C] — channel dim is the small one + is_hwc = (pv_shape[2] < pv_shape[0]); + if (is_hwc) { + height = pv_shape[0]; + width = pv_shape[1]; + channels = pv_shape[2]; + } else { + channels = pv_shape[0]; + height = pv_shape[1]; + width = pv_shape[2]; + } + } else if (pv_ndims == 4) { + num_imgs = pv_shape[0]; + is_hwc = (pv_shape[3] < pv_shape[1]); + if (is_hwc) { + height = pv_shape[1]; + width = pv_shape[2]; + channels = pv_shape[3]; + } else { + channels = pv_shape[1]; + height = pv_shape[2]; + width = pv_shape[3]; + } + } else { + throw std::runtime_error("VideoChatFlashProcessor: unexpected pixel_values rank " + + std::to_string(pv_ndims) + " (expected 3 or 4)"); + } + + // Vision model expects [1, num_frames, C, H, W] + { + std::vector target_shape = {1, num_imgs, channels, height, width}; + size_t count = static_cast(num_imgs * channels * height * width); + + auto float_tensor = OrtValue::CreateTensor(allocator, target_shape); + float* dst = float_tensor->GetTensorMutableData(); + + if (is_hwc) { + for (int64_t n = 0; n < num_imgs; ++n) { + const float* src_img = pv_data + n * height * width * channels; + float* dst_img = dst + n * channels * height * width; + for (int64_t c = 0; c < channels; ++c) + for (int64_t h = 0; h < height; ++h) + for (int64_t w = 0; w < width; ++w) + dst_img[c * height * width + h * width + w] = src_img[h * width * channels + w * channels + c]; + } + } else { + std::copy(pv_data, pv_data + count, dst); + } + + std::unique_ptr pv_ortvalue; + if (pixel_values_type_ == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { + pv_ortvalue = std::move(float_tensor); + } else { + auto* p_device = GetDeviceInterface(DeviceType::CPU); + Cast(*float_tensor, pv_ortvalue, *p_device, pixel_values_type_); + } + named_tensors->emplace(std::string(Config::Defaults::PixelValuesName), + std::make_shared(std::move(pv_ortvalue))); + } + + // Tokenize prompt with fixed visual token padding + auto [input_ids, num_img_tokens] = BuildPromptTokens( + tokenizer, prompt, static_cast(images->num_images_), + num_visual_tokens_, allocator); + named_tensors->emplace(std::string(Config::Defaults::InputIdsName), + std::make_shared(std::move(input_ids))); + named_tensors->emplace(std::string(Config::Defaults::NumImageTokens), + std::make_shared(std::move(num_img_tokens))); + + // Emit image_grid_thw for GetImageFeatureBatchSize to determine num_images. + // The pixel_values name is remapped (e.g. "pixel_values" → "images"), so the + // rank-based lookup in GetImageFeatureBatchSize won't match; it falls through + // to image_grid_thw whose name is not remapped. + auto grid_thw = OrtValue::CreateTensor(allocator, std::vector{num_imgs, 3}); + auto* grid_ptr = grid_thw->GetTensorMutableData(); + for (int64_t i = 0; i < num_imgs; ++i) { + grid_ptr[i * 3 + 0] = 1; + grid_ptr[i * 3 + 1] = height; + grid_ptr[i * 3 + 2] = width; + } + named_tensors->emplace(std::string(Config::Defaults::ImageGridThwName), + std::make_shared(std::move(grid_thw))); + + return named_tensors; +} + +} // namespace Generators diff --git a/src/models/videochat_flash_processor.h b/src/models/videochat_flash_processor.h new file mode 100644 index 0000000000..75ca17e492 --- /dev/null +++ b/src/models/videochat_flash_processor.h @@ -0,0 +1,29 @@ +// Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. +// +// SPDX-License-Identifier: MIT +// +// Licensed under the MIT License. See License.txt in the project root for +// license information. + +#pragma once + +#include "model.h" +#include "processor.h" +#include "ortx_processor.h" + +namespace Generators { + +struct VideoChatFlashProcessor : Processor { + VideoChatFlashProcessor(Config& config, const SessionInfo& session_info); + + std::unique_ptr Process(const Tokenizer& tokenizer, const Payload& payload) const override; + + private: + ort_extensions::OrtxObjectPtr processor_; + + ONNXTensorElementDataType pixel_values_type_; + int64_t num_visual_tokens_; +}; + +} // namespace Generators diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index 8eb68366fc..fb1a65eae8 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -3,7 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. Portions of this file consist of AI generated content. +# Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# Portions of this file consist of AI generated content. # -------------------------------------------------------------------------- """ Run the model builder to create the desired ONNX model. @@ -46,6 +47,7 @@ Qwen35TextModel, QwenModel, SmolLM3Model, + VideoChatFlashQwenModel, WhisperModel, ) from transformers import ( @@ -289,6 +291,10 @@ def create_model( onnx_model = Phi4MMModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "Qwen2ForCausalLM": onnx_model = QwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) + elif config.architectures[0] == "VideoChatFlashQwenForCausalLM": + print("WARNING: This is only generating the text component of the model. Setting `--extra_options exclude_embeds=true` by default.") + extra_options["exclude_embeds"] = True + onnx_model = VideoChatFlashQwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "Qwen2_5_VLForConditionalGeneration": text_config = config.text_config for key in text_config: diff --git a/src/python/py/models/builders/__init__.py b/src/python/py/models/builders/__init__.py index 4e4d1ae93e..3522467dda 100644 --- a/src/python/py/models/builders/__init__.py +++ b/src/python/py/models/builders/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # ------------------------------------------------------------------------- -# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +# Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. # Portions of this file consist of AI generated content. # ------------------------------------------------------------------------- from .base import Model @@ -28,7 +28,7 @@ Phi4MMModel, PhiModel, ) -from .qwen import Qwen3Model, Qwen3VLTextModel, Qwen25VLTextModel, Qwen35TextModel, QwenModel +from .qwen import Qwen3Model, Qwen25VLTextModel, Qwen3VLTextModel, Qwen35TextModel, QwenModel, VideoChatFlashQwenModel from .smollm import SmolLM3Model from .whisper import WhisperModel @@ -62,5 +62,6 @@ "Qwen35TextModel", "QwenModel", "SmolLM3Model", + "VideoChatFlashQwenModel", "WhisperModel", ] diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index b33680ce4a..2d7adf163b 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -3,13 +3,17 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +# Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# Portions of this file consist of AI generated content. +import os import numpy as np import onnx_ir as ir import torch from transformers import ( AutoConfig, + Qwen2ForCausalLM, Qwen2_5_VLForConditionalGeneration, Qwen3VLForConditionalGeneration, ) @@ -913,6 +917,35 @@ def load_weights(self, input_path): ) +class VideoChatFlashQwenModel(QwenModel): + """ + Builder for OpenGVLab/VideoChat-Flash models (VideoChatFlashQwenForCausalLM). + + The language model backbone is standard Qwen2.5-7B with flat config and + standard weight keys (model.layers.*, lm_head.*). The model uses standard + 2D RoPE (rope_scaling=None) and GQA (28 query heads, 4 KV heads). + + This builder exports only the text decoder component. It sets exclude_embeds=True + so the decoder receives inputs_embeds from the embedding merger model, which + fuses the InternVideo2 visual tokens with text embeddings. + """ + + def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): + super().__init__(config, io_dtype, onnx_dtype, ep, cache_dir, extra_options) + + # Override model_type for the C++ runtime registration in model.cpp + # and genai_config.json. Same pattern as Qwen3VLTextModel. + # Base class transforms this to "videochat_flash_qwen" via: + # model_type[:model_type.find("For")].lower() + self.model_type = "VideoChat_Flash_QwenForCausalLM" + + def load_weights(self, input_path): + extra_kwargs = {} if os.path.isdir(self.model_name_or_path) else {"cache_dir": self.cache_dir} + return Qwen2ForCausalLM.from_pretrained( + self.model_name_or_path, + token=self.hf_token, + **extra_kwargs, + ) class Qwen35TextModel(Model): """Qwen3.5 hybrid model builder.