From c3c0f82cabd2e9b6e27be91394e5a3946acf30f0 Mon Sep 17 00:00:00 2001 From: Rajeev Patwari Date: Tue, 24 Mar 2026 17:53:10 -0700 Subject: [PATCH 01/22] Add VideoChat-Flash (OpenGVLab) language model support Adds text decoder export support for OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B. Architecture: VideoChatFlashQwenForCausalLM is a VLM whose language backbone is standard Qwen2.5-7B (flat config, standard weight keys, 2D RoPE, rope_theta=1e6, 28L / 28h / 4kv / hidden=3584). The model does NOT use MRoPE, so the builder inherits QwenModel directly rather than Qwen25VLTextModel. Changes: - src/python/py/models/builders/qwen.py: Add VideoChatFlashQwenModel subclass of QwenModel. Sets exclude_embeds=True (text decoder receives inputs_embeds from the embedding merger) and model_type="videochat_flash_qwen". - src/python/py/models/builders/__init__.py: Export VideoChatFlashQwenModel. - src/python/py/models/builder.py: Map "VideoChatFlashQwenForCausalLM" architecture string to VideoChatFlashQwenModel with exclude_embeds=True. - src/models/model_type.h: Register "videochat_flash_qwen" in IsVLM() (size 6->7). - examples/python/videochat-flash/builder.py: New example export script. Phase 1 (this PR): text decoder only. Vision encoder (InternVideo2-1B) and embedding merger export are Phase 2 (scaffolded as TODOs). Co-Authored-By: Claude Sonnet 4.6 --- examples/python/videochat-flash/builder.py | 258 +++++++++++++++++++++ src/models/model_type.h | 2 +- src/python/py/models/builder.py | 5 + src/python/py/models/builders/__init__.py | 3 +- src/python/py/models/builders/qwen.py | 18 ++ 5 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 examples/python/videochat-flash/builder.py diff --git a/examples/python/videochat-flash/builder.py b/examples/python/videochat-flash/builder.py new file mode 100644 index 0000000000..7a4b4c2443 --- /dev/null +++ b/examples/python/videochat-flash/builder.py @@ -0,0 +1,258 @@ +# ------------------------------------------------------------------------- +# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +# Portions of this file consist of AI generated content. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +""" +Export VideoChat-Flash (OpenGVLab) ONNX models for onnxruntime-genai. + +Model: OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B +Architecture: + - Language backbone: Qwen2.5-7B (28L, GQA 28h/4kv, hidden=3584) + - Visual encoder: InternVideo2-1B (video ViT with 3D spatiotemporal attention) + - Connector: MLP-based HiCo token compression (~16 tokens/frame) + +This script exports: + 1. Text decoder (model.onnx) via OGA builder — fully functional + 2. Vision encoder (vcf-vision.onnx) — TODO: InternVideo2 export + 3. Embedding merger (vcf-embedding.onnx) — TODO: token fusion export + 4. genai_config.json — wired for OGA multimodal pipeline + +Phase 1 (this PR): Text decoder only. Vision/embedding stubs are included +as placeholders. Pass --text-only to skip vision/embedding export. + +Usage: + # Download model and export text decoder only (Phase 1): + python builder.py --output ./vcf-oga-fp32 --text-only + + # Full pipeline export (requires vision encoder work): + python builder.py --input ./pytorch_vcf --output ./vcf-oga-int4 + + # Text-only inference smoke test after export: + python builder.py --output ./vcf-oga-fp32 --text-only --run-e2e +""" + +import argparse +import json +import os + +import torch +import torch.nn as nn + +from onnxruntime_genai.models.builder import create_model + +HF_MODEL_ID = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" + +# VideoChat-Flash image placeholder token id (same vocab as Qwen2.5) +IMAGE_TOKEN_ID = 151646 # <|image_pad|> — verify against tokenizer_config.json + + +def prepare_model(input_dir): + """Load HF model from local path or HuggingFace.""" + from transformers import AutoConfig + + print("\n[1/4] Loading model config...") + config = AutoConfig.from_pretrained(input_dir, trust_remote_code=True) + print(f" architecture : {config.architectures[0]}") + print(f" hidden_size : {config.hidden_size}") + print(f" num_layers : {config.num_hidden_layers}") + print(f" num_heads : {config.num_attention_heads} Q / {config.num_key_value_heads} KV") + print(f" rope_theta : {config.rope_theta}") + print(f" vocab_size : {config.vocab_size}") + return config + + +def export_vision_model(model, config, output_dir): + """ + Export the InternVideo2-1B visual encoder. + + TODO (Phase 2): The InternVideo2 encoder uses 3D spatiotemporal attention + and learnable position embeddings. Export requires: + 1. Wrapping model.vision_tower (InternVideo2) in a torch.onnx-compatible + nn.Module that accepts (pixel_values: [T*H*W, C], grid_thw: [N, 3]). + 2. Exporting the HiCo clip-level MLP compression head. + 3. Verifying opset compatibility for temporal attention ops. + + For now this function is a no-op placeholder. + """ + print("\n[2/4] Vision encoder export — TODO (Phase 2, skipped)") + print(" The InternVideo2-1B encoder requires custom 3D spatiotemporal") + print(" attention export. See Phase 2 implementation plan.") + + +def export_embedding_model(model, config, output_dir): + """ + Export the embedding merger that fuses visual tokens into the token stream. + + TODO (Phase 2): VideoChat-Flash uses mm_patch_merge_type and llm_compress + layers to inject visual tokens. Export requires: + 1. Wrapping model.model.embed_tokens (Qwen2.5 embedding table). + 2. Implementing the token replacement mask (image_token_id → vision features). + 3. Handling the HiCo video-level compression in LLM layers (llm_compress_layer_list). + + For now this function is a no-op placeholder. + """ + print("\n[3/4] Embedding merger export — TODO (Phase 2, skipped)") + print(" Token fusion depends on Phase 2 vision encoder output shape.") + + +def export_text_model(input_dir, output_dir, precision): + """Export text decoder (Qwen2.5-7B backbone) via OGA builder.""" + print(f"\n[{3 if False else 2}/4] Exporting text decoder ({precision.upper()})...") + print(f" Source: {input_dir}") + + create_model( + HF_MODEL_ID if input_dir is None else input_dir, + input_dir, + output_dir, + precision, + "cpu", + os.path.join(output_dir, ".cache"), + ) + print(f" [OK] Text decoder: {os.path.join(output_dir, 'model.onnx')}") + + +def update_genai_config(output_dir, text_only): + """Patch genai_config.json with VideoChat-Flash model type and vision sections.""" + config_path = os.path.join(output_dir, "genai_config.json") + + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + config["model"]["type"] = "videochat_flash_qwen" + + if not text_only: + # Phase 2: wire vision encoder and embedding merger + config["model"]["vision"] = { + "filename": "vcf-vision.onnx", + "inputs": { + "pixel_values": "pixel_values", + "image_grid_thw": "image_grid_thw", + }, + "outputs": { + "image_features": "visual_tokens", + }, + } + config["model"]["embedding"] = { + "filename": "vcf-embedding.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "visual_tokens", + }, + "outputs": { + "inputs_embeds": "inputs_embeds", + }, + } + + with open(config_path, "w", encoding="utf-8") as f: + json.dump(config, f, indent=2) + + print(f" [OK] Updated: genai_config.json (type=videochat_flash_qwen)") + + +def run_e2e_smoke(output_dir, prompt): + """Quick text-only inference smoke test.""" + import onnxruntime_genai as og + + print("\n[Smoke] Running text-only inference...") + model = og.Model(output_dir) + tokenizer = og.Tokenizer(model) + tokens = tokenizer.encode(prompt) + + params = og.GeneratorParams(model) + params.set_search_options(max_length=128) + params.input_ids = tokens + + generator = og.Generator(model, params) + print("Output:", end=" ", flush=True) + while not generator.is_done(): + generator.generate_next_token() + token = generator.get_next_tokens()[0] + print(tokenizer.decode([token]), end="", flush=True) + print() + + +def main(): + parser = argparse.ArgumentParser( + description="Export VideoChat-Flash for onnxruntime-genai" + ) + parser.add_argument( + "--input", + type=str, + default=None, + help="Local PyTorch model directory. If omitted, downloads from HuggingFace.", + ) + parser.add_argument( + "--output", + type=str, + default="./vcf-oga", + help="Output directory for ONNX models", + ) + parser.add_argument( + "--precision", + type=str, + default="fp32", + choices=["fp32", "fp16", "int4"], + help="Text decoder precision", + ) + parser.add_argument( + "--text-only", + action="store_true", + help="Export text decoder only (Phase 1). Skip vision/embedding export.", + ) + parser.add_argument( + "--run-e2e", + action="store_true", + help="Run text-only inference smoke test after export.", + ) + parser.add_argument( + "--prompt", + type=str, + default="Describe the video in one sentence.", + help="Prompt for --run-e2e smoke test", + ) + args = parser.parse_args() + + input_dir = args.input or HF_MODEL_ID + output_dir = os.path.abspath(args.output) + os.makedirs(output_dir, exist_ok=True) + + print("=" * 70) + print("VideoChat-Flash ONNX Export for OGA") + print("=" * 70) + print(f" Source : {input_dir}") + print(f" Output : {output_dir}") + print(f" Precision: {args.precision.upper()}") + print(f" Mode : {'text-only (Phase 1)' if args.text_only else 'full pipeline (Phase 2)'}") + + prepare_model(input_dir) + + if not args.text_only: + # Phase 2: load full model weights for vision/embedding export + export_vision_model(None, None, output_dir) + export_embedding_model(None, None, output_dir) + + export_text_model(input_dir, output_dir, args.precision) + + print("\n[4/4] Updating genai_config.json...") + update_genai_config(output_dir, args.text_only) + + if args.run_e2e: + run_e2e_smoke(output_dir, args.prompt) + + print("\n" + "=" * 70) + print("[SUCCESS] Export complete!") + print("=" * 70) + print(f"\nOutput: {output_dir}") + print("\nExported files:") + print(f" model.onnx ({args.precision.upper()}, text decoder — Qwen2.5-7B)") + if not args.text_only: + print(" vcf-vision.onnx (Phase 2 TODO — InternVideo2-1B)") + print(" vcf-embedding.onnx (Phase 2 TODO — token merger)") + print(" genai_config.json (type=videochat_flash_qwen)") + print() + + +if __name__ == "__main__": + main() diff --git a/src/models/model_type.h b/src/models/model_type.h index 83ac34e8cd..8cf387bd9c 100644 --- a/src/models/model_type.h +++ b/src/models/model_type.h @@ -21,7 +21,7 @@ struct ModelType { inline static bool IsVLM(const std::string& model_type) { // Vision-language model (VLM) - static constexpr std::array VLM = {"fara", "gemma3", "phi3v", "qwen2_5_vl", "qwen3_vl", "qwen3_5"}; + static constexpr std::array VLM = {"fara", "gemma3", "phi3v", "qwen2_5_vl", "qwen3_vl", "qwen3_5", "videochat_flash_qwen"}; return std::find(VLM.begin(), VLM.end(), model_type) != VLM.end(); } diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index eb0dc0bb85..af4775db79 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -42,6 +42,7 @@ Qwen25VLTextModel, Qwen3VLTextModel, QwenModel, + VideoChatFlashQwenModel, SmolLM3Model, WhisperModel, ) @@ -278,6 +279,10 @@ def create_model( onnx_model = Phi4MMModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "Qwen2ForCausalLM": onnx_model = QwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) + elif config.architectures[0] == "VideoChatFlashQwenForCausalLM": + print("WARNING: This is only generating the text component of the model. Setting `--extra_options exclude_embeds=true` by default.") + extra_options["exclude_embeds"] = True + onnx_model = VideoChatFlashQwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "Qwen2_5_VLForConditionalGeneration": text_config = config.text_config for key in text_config: diff --git a/src/python/py/models/builders/__init__.py b/src/python/py/models/builders/__init__.py index 03be8ef71d..2ec9e7667e 100644 --- a/src/python/py/models/builders/__init__.py +++ b/src/python/py/models/builders/__init__.py @@ -27,7 +27,7 @@ Phi4MMModel, PhiModel, ) -from .qwen import Qwen3Model, Qwen25VLTextModel, Qwen3VLTextModel, QwenModel +from .qwen import Qwen3Model, Qwen25VLTextModel, Qwen3VLTextModel, QwenModel, VideoChatFlashQwenModel from .smollm import SmolLM3Model from .whisper import WhisperModel @@ -57,6 +57,7 @@ "Qwen3VLTextModel", "Qwen25VLTextModel", "QwenModel", + "VideoChatFlashQwenModel", "SmolLM3Model", "WhisperModel", ] diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 69a6f48f82..8561feb565 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -911,3 +911,21 @@ def load_weights(self, input_path): token=self.hf_token, trust_remote_code=self.hf_remote, ) + + +class VideoChatFlashQwenModel(QwenModel): + """ + Builder for OpenGVLab/VideoChat-Flash models (VideoChatFlashQwenForCausalLM). + + The language model backbone is standard Qwen2.5-7B with flat config and + standard weight keys (model.layers.*, lm_head.*). The model uses standard + 2D RoPE (rope_scaling=None) and GQA (28 query heads, 4 KV heads). + + This builder exports only the text decoder component. It sets exclude_embeds=True + so the decoder receives inputs_embeds from the embedding merger model, which + fuses the InternVideo2 visual tokens with text embeddings. + """ + + def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): + super().__init__(config, io_dtype, onnx_dtype, ep, cache_dir, extra_options) + self.model_type = "videochat_flash_qwen" From 88a7d3769c6e71be9184c8d1fa1fd0962c957084 Mon Sep 17 00:00:00 2001 From: Rajeev Patwari Date: Tue, 24 Mar 2026 20:37:14 -0700 Subject: [PATCH 02/22] Fix VideoChatFlash export: bypass video library deps, add standalone inference support - builder.py (example): use local OGA builder via sys.path; fix create_model args when no local input dir; move prepare_model() inside Phase-2 block (trust_remote_code=False is safe for config-only loads) - builder.py (core): add VCF config bypass using hf_hub_download + Qwen2Config to avoid av/cv2/decord/imageio imports triggered by AutoConfig; set config._name_or_path so load_weights() resolves the model correctly; make exclude_embeds opt-in (guard with 'not in extra_options') so standalone (input_ids) export also works - qwen.py: add make_genai_config() override that writes a temp Qwen2Config and patches genai_config.json type back to videochat_flash_qwen; add load_weights() override using Qwen2ForCausalLM.from_pretrained() directly Validated: text-only inference with vcf-oga-fp32-standalone/ produces correct answers (Paris, 56, ONNX Runtime description) using append_tokens() API on the exported Qwen2.5-7B backbone. Co-Authored-By: Claude Sonnet 4.6 --- examples/python/videochat-flash/builder.py | 27 ++++++++---- src/python/py/models/builder.py | 34 +++++++++++++-- src/python/py/models/builders/qwen.py | 50 ++++++++++++++++++++++ 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/examples/python/videochat-flash/builder.py b/examples/python/videochat-flash/builder.py index 7a4b4c2443..e853b67d28 100644 --- a/examples/python/videochat-flash/builder.py +++ b/examples/python/videochat-flash/builder.py @@ -36,11 +36,14 @@ import argparse import json import os +import sys -import torch -import torch.nn as nn +# Use the local OGA model builder (src/python/py/models/builder.py) +# so our VideoChatFlashQwenModel registration is picked up. +_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +sys.path.insert(0, os.path.join(_REPO_ROOT, "src", "python", "py", "models")) -from onnxruntime_genai.models.builder import create_model +from builder import create_model # noqa: E402 (local OGA builder) HF_MODEL_ID = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" @@ -49,11 +52,13 @@ def prepare_model(input_dir): - """Load HF model from local path or HuggingFace.""" + """Load HF model config from local path or HuggingFace.""" from transformers import AutoConfig print("\n[1/4] Loading model config...") - config = AutoConfig.from_pretrained(input_dir, trust_remote_code=True) + # trust_remote_code=False: config.json is standard JSON — no need for + # the custom modeling code (which requires av/cv2/decord). + config = AutoConfig.from_pretrained(input_dir, trust_remote_code=False) print(f" architecture : {config.architectures[0]}") print(f" hidden_size : {config.hidden_size}") print(f" num_layers : {config.num_hidden_layers}") @@ -102,9 +107,14 @@ def export_text_model(input_dir, output_dir, precision): print(f"\n[{3 if False else 2}/4] Exporting text decoder ({precision.upper()})...") print(f" Source: {input_dir}") + # create_model(model_name, input_path, ...): + # model_name — HF repo ID used for config/tokenizer lookup + # input_path — local dir (or HF repo ID when downloading) + hf_id = HF_MODEL_ID + local_or_hf = input_dir if input_dir is not None else hf_id create_model( - HF_MODEL_ID if input_dir is None else input_dir, - input_dir, + hf_id, + local_or_hf, output_dir, precision, "cpu", @@ -226,10 +236,9 @@ def main(): print(f" Precision: {args.precision.upper()}") print(f" Mode : {'text-only (Phase 1)' if args.text_only else 'full pipeline (Phase 2)'}") - prepare_model(input_dir) - if not args.text_only: # Phase 2: load full model weights for vision/embedding export + prepare_model(input_dir) export_vision_model(None, None, output_dir) export_embedding_model(None, None, output_dir) diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index af4775db79..0d623a0b39 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -192,7 +192,34 @@ def create_model( hf_token = parse_hf_token(extra_options.get("hf_token", "true")) hf_remote = extra_options.get("hf_remote", True) - config = AutoConfig.from_pretrained(hf_name, token=hf_token, trust_remote_code=hf_remote, **extra_kwargs) + # VideoChat-Flash uses custom remote code that imports heavy video libraries + # (av, cv2, decord, imageio) even though the LM backbone is standard Qwen2.5. + # Load the raw config.json via Qwen2Config to avoid pulling in video deps. + _vcf_arch = "VideoChatFlashQwenForCausalLM" + _is_vcf = False + try: + import json as _json + if os.path.isdir(hf_name): + _raw_cfg_path = os.path.join(hf_name, "config.json") + if os.path.isfile(_raw_cfg_path): + with open(_raw_cfg_path) as _f: + _is_vcf = _json.load(_f).get("architectures", [None])[0] == _vcf_arch + else: + # HF repo: peek at config.json without running custom code + from huggingface_hub import hf_hub_download + _cfg_file = hf_hub_download(repo_id=hf_name, filename="config.json", token=hf_token, cache_dir=cache_dir) + with open(_cfg_file) as _f: + _is_vcf = _json.load(_f).get("architectures", [None])[0] == _vcf_arch + except Exception: + pass + + if _is_vcf: + from transformers import Qwen2Config + config = Qwen2Config.from_pretrained(hf_name, token=hf_token, **extra_kwargs) + config.architectures = [_vcf_arch] + config._name_or_path = hf_name # ensure load_weights can find the weights + else: + config = AutoConfig.from_pretrained(hf_name, token=hf_token, trust_remote_code=hf_remote, **extra_kwargs) if "adapter_path" in extra_options: from peft import PeftConfig @@ -280,8 +307,9 @@ def create_model( elif config.architectures[0] == "Qwen2ForCausalLM": onnx_model = QwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "VideoChatFlashQwenForCausalLM": - print("WARNING: This is only generating the text component of the model. Setting `--extra_options exclude_embeds=true` by default.") - extra_options["exclude_embeds"] = True + if "exclude_embeds" not in extra_options: + print("WARNING: This is only generating the text component of the model. Setting `--extra_options exclude_embeds=true` by default.") + extra_options["exclude_embeds"] = True onnx_model = VideoChatFlashQwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "Qwen2_5_VLForConditionalGeneration": text_config = config.text_config diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 8561feb565..d84cdef9ab 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -5,6 +5,8 @@ # -------------------------------------------------------------------------- +import os + import onnx_ir as ir import torch from transformers import Qwen2_5_VLForConditionalGeneration, Qwen3VLForConditionalGeneration @@ -929,3 +931,51 @@ class VideoChatFlashQwenModel(QwenModel): def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): super().__init__(config, io_dtype, onnx_dtype, ep, cache_dir, extra_options) self.model_type = "videochat_flash_qwen" + # The custom remote code requires video libraries (av, cv2, decord, imageio) + # which are not needed to export the LM backbone. Disable trust_remote_code + # so base class helpers (make_genai_config, save_processing) use standard paths. + self.hf_remote = False + + def make_genai_config(self, model_name_or_path, extra_kwargs, out_dir): + # make_genai_config in base.py calls AutoConfig with trust_remote_code, + # which triggers the video library imports. Instead, write a clean + # Qwen2-compatible config.json to a temp dir and let base class read it. + import json as _json + import shutil + import tempfile + + from transformers import Qwen2Config + + vcf_config = Qwen2Config.from_pretrained(model_name_or_path, token=self.hf_token, **extra_kwargs) + vcf_config.architectures = ["VideoChatFlashQwenForCausalLM"] + vcf_config.model_type = "videochat_flash_qwen" + vcf_config._name_or_path = model_name_or_path + + tmp_dir = tempfile.mkdtemp() + try: + with open(os.path.join(tmp_dir, "config.json"), "w") as f: + _json.dump(vcf_config.to_dict(), f) + super().make_genai_config(tmp_dir, {}, out_dir) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + # Restore the correct model type (base class may write "qwen2" from Qwen2Config) + gcfg_path = os.path.join(out_dir, "genai_config.json") + if os.path.isfile(gcfg_path): + with open(gcfg_path) as f: + gcfg = _json.load(f) + gcfg["model"]["type"] = "videochat_flash_qwen" + with open(gcfg_path, "w") as f: + _json.dump(gcfg, f, indent=2) + + def load_weights(self, input_path): + # The LM backbone is identical to Qwen2ForCausalLM. Load it directly + # to avoid the custom remote code (which requires video libraries). + from transformers import Qwen2ForCausalLM + print("Loading VideoChatFlash model as Qwen2ForCausalLM...") + extra_kwargs = {} if os.path.isdir(self.model_name_or_path) else {"cache_dir": self.cache_dir} + return Qwen2ForCausalLM.from_pretrained( + self.model_name_or_path, + token=self.hf_token, + **extra_kwargs, + ) From 5c32743c9e41c237b091b346261ca1646baeaabf Mon Sep 17 00:00:00 2001 From: Rajeev Patwari Date: Tue, 24 Mar 2026 20:50:14 -0700 Subject: [PATCH 03/22] Add VideoChat-Flash text inference test script (run.py) examples/python/videochat-flash/run.py: - Text-only inference for the exported OGA model - Uses HF AutoTokenizer (og.Tokenizer fails for TokenizersBackend models) - Feeds tokens via generator.append_tokens(np.int32 array) - --batch flag runs 4 built-in QA prompts for quick validation - --prompt / --max-length for custom single-prompt runs - Notes in docstring on genai_config.json type patch workaround for installed OGA binaries that predate videochat_flash_qwen support Validated: all 4 batch prompts produce correct answers on vcf-oga-fp32-standalone. Co-Authored-By: Claude Sonnet 4.6 --- examples/python/videochat-flash/run.py | 122 +++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 examples/python/videochat-flash/run.py diff --git a/examples/python/videochat-flash/run.py b/examples/python/videochat-flash/run.py new file mode 100644 index 0000000000..b0439c18c1 --- /dev/null +++ b/examples/python/videochat-flash/run.py @@ -0,0 +1,122 @@ +# ------------------------------------------------------------------------- +# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +# Portions of this file consist of AI generated content. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +""" +Text-only inference test for VideoChat-Flash (OpenGVLab) exported via OGA. + +Usage: + # After exporting with builder.py --text-only: + python run.py --model ./vcf-oga-fp32-standalone + + # Custom prompt: + python run.py --model ./vcf-oga-fp32-standalone --prompt "Explain ONNX in one sentence." + +Notes: + - Requires the model to be exported WITHOUT exclude_embeds (i.e. with + `--extra_options exclude_embeds=false` or the standalone export mode). + - The installed OGA binary may not recognise 'videochat_flash_qwen' as a + model type. If you see an unsupported-model-type error, patch + genai_config.json to set "type": "qwen2" — the LM backbone is identical. + - Uses HF AutoTokenizer directly (og.Tokenizer may fail for this model). +""" + +import argparse +import os + +import numpy as np + +HF_MODEL_ID = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" + +# Chat-ML template tokens (Qwen-style) +_IM_START = "<|im_start|>" +_IM_END = "<|im_end|>" + + +def build_prompt(user_text: str) -> str: + return f"{_IM_START}user\n{user_text}{_IM_END}\n{_IM_START}assistant\n" + + +def run_inference(model_dir: str, prompt: str, max_length: int = 256) -> str: + from onnxruntime_genai import onnxruntime_genai as og # noqa: PLC0415 + from transformers import AutoTokenizer # noqa: PLC0415 + + print(f"[1/3] Loading tokenizer from HuggingFace ({HF_MODEL_ID})...") + tok = AutoTokenizer.from_pretrained(HF_MODEL_ID, trust_remote_code=False) + + print(f"[2/3] Loading OGA model from {model_dir}...") + model = og.Model(model_dir) + + formatted = build_prompt(prompt) + input_ids = np.array(tok.encode(formatted), dtype=np.int32) + print(f"[3/3] Generating (input={len(input_ids)} tokens, max_length={max_length})...") + + params = og.GeneratorParams(model) + params.set_search_options(max_length=max_length, do_sample=False) + generator = og.Generator(model, params) + generator.append_tokens(input_ids) + + output_tokens = [] + while not generator.is_done(): + generator.generate_next_token() + tok_id = int(generator.get_next_tokens()[0]) + output_tokens.append(tok_id) + + return tok.decode(output_tokens, skip_special_tokens=True) + + +def main(): + parser = argparse.ArgumentParser( + description="Text-only inference test for VideoChat-Flash OGA export" + ) + parser.add_argument( + "--model", + type=str, + required=True, + help="Path to exported OGA model directory (e.g. ./vcf-oga-fp32-standalone)", + ) + parser.add_argument( + "--prompt", + type=str, + default="What is the capital of France? Give a short answer.", + help="User prompt (plain text, chat template applied automatically)", + ) + parser.add_argument( + "--max-length", + type=int, + default=256, + help="Maximum number of tokens to generate", + ) + parser.add_argument( + "--batch", + action="store_true", + help="Run a small batch of built-in test prompts instead of --prompt", + ) + args = parser.parse_args() + + model_dir = os.path.abspath(args.model) + + if args.batch: + test_prompts = [ + "What is the capital of France? Give a short answer.", + "What is 7 * 8?", + "Explain what ONNX Runtime is in 2 sentences.", + "Name three primary colors.", + ] + else: + test_prompts = [args.prompt] + + for i, prompt in enumerate(test_prompts, 1): + print(f"\n{'='*60}") + print(f"[{i}/{len(test_prompts)}] Prompt: {prompt}") + print("=" * 60) + response = run_inference(model_dir, prompt, args.max_length) + print(f"Response: {response}") + + print("\n[OK] Inference complete.") + + +if __name__ == "__main__": + main() From 5fbd066ce8aa1f91477d92174acd2bec18956fec Mon Sep 17 00:00:00 2001 From: Rajeev Patwari Date: Wed, 25 Mar 2026 08:03:04 -0700 Subject: [PATCH 04/22] Fix standalone export and inference: use qwen2 type, input_ids for text-only mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of teammate's inference failure: 1. builder.py set type=videochat_flash_qwen even in --text-only mode. OGA loads this via MultiModalLanguageModel which requires vision.onnx + embedding.onnx (not present in text-only export) → 'File doesn't exist' error on og.Model(). 2. exclude_embeds defaulted to true → decoder expected inputs_embeds, but run.py feeds token IDs via append_tokens() → 'input not found' error. Fix: - builder.py export_text_model(): pass exclude_embeds=false in text-only mode (decoder uses input_ids for standalone inference). - builder.py update_genai_config(): set type=qwen2 in text-only mode so OGA loads the model as a plain decoder (LM backbone is identical to Qwen2.5-7B). type=videochat_flash_qwen is reserved for Phase 2 full VLM pipeline. - run.py: simplify docstring now that no manual patching is needed. Validated with compiled OGA build (onnxruntime-genai conda env, Python 3.11): all 4 batch prompts produce correct answers without any manual config changes. Co-Authored-By: Claude Sonnet 4.6 --- examples/python/videochat-flash/builder.py | 17 ++++++++++++++--- examples/python/videochat-flash/run.py | 7 ++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/examples/python/videochat-flash/builder.py b/examples/python/videochat-flash/builder.py index e853b67d28..8ba915dcb0 100644 --- a/examples/python/videochat-flash/builder.py +++ b/examples/python/videochat-flash/builder.py @@ -102,7 +102,7 @@ def export_embedding_model(model, config, output_dir): print(" Token fusion depends on Phase 2 vision encoder output shape.") -def export_text_model(input_dir, output_dir, precision): +def export_text_model(input_dir, output_dir, precision, text_only): """Export text decoder (Qwen2.5-7B backbone) via OGA builder.""" print(f"\n[{3 if False else 2}/4] Exporting text decoder ({precision.upper()})...") print(f" Source: {input_dir}") @@ -110,6 +110,9 @@ def export_text_model(input_dir, output_dir, precision): # create_model(model_name, input_path, ...): # model_name — HF repo ID used for config/tokenizer lookup # input_path — local dir (or HF repo ID when downloading) + # exclude_embeds controls decoder input: + # text_only → False (input_ids, standalone text inference) + # full VLM → True (inputs_embeds, from embedding merger) hf_id = HF_MODEL_ID local_or_hf = input_dir if input_dir is not None else hf_id create_model( @@ -119,6 +122,7 @@ def export_text_model(input_dir, output_dir, precision): precision, "cpu", os.path.join(output_dir, ".cache"), + exclude_embeds="false" if text_only else "true", ) print(f" [OK] Text decoder: {os.path.join(output_dir, 'model.onnx')}") @@ -130,7 +134,14 @@ def update_genai_config(output_dir, text_only): with open(config_path, "r", encoding="utf-8") as f: config = json.load(f) - config["model"]["type"] = "videochat_flash_qwen" + if text_only: + # Text-only / standalone mode: decoder takes input_ids, no vision/embedding. + # Use qwen2 type so OGA loads it as a plain decoder (MultiModalLanguageModel + # requires vision.onnx + embedding.onnx which are not present in this mode). + config["model"]["type"] = "qwen2" + else: + # Full VLM pipeline: decoder takes inputs_embeds from the embedding merger. + config["model"]["type"] = "videochat_flash_qwen" if not text_only: # Phase 2: wire vision encoder and embedding merger @@ -242,7 +253,7 @@ def main(): export_vision_model(None, None, output_dir) export_embedding_model(None, None, output_dir) - export_text_model(input_dir, output_dir, args.precision) + export_text_model(input_dir, output_dir, args.precision, args.text_only) print("\n[4/4] Updating genai_config.json...") update_genai_config(output_dir, args.text_only) diff --git a/examples/python/videochat-flash/run.py b/examples/python/videochat-flash/run.py index b0439c18c1..96440917a6 100644 --- a/examples/python/videochat-flash/run.py +++ b/examples/python/videochat-flash/run.py @@ -15,11 +15,8 @@ python run.py --model ./vcf-oga-fp32-standalone --prompt "Explain ONNX in one sentence." Notes: - - Requires the model to be exported WITHOUT exclude_embeds (i.e. with - `--extra_options exclude_embeds=false` or the standalone export mode). - - The installed OGA binary may not recognise 'videochat_flash_qwen' as a - model type. If you see an unsupported-model-type error, patch - genai_config.json to set "type": "qwen2" — the LM backbone is identical. + - builder.py --text-only exports a standalone decoder with input_ids and + genai_config.json type=qwen2 (compatible with all OGA binary versions). - Uses HF AutoTokenizer directly (og.Tokenizer may fail for this model). """ From 356c1aece7b262141beed3e1df64a75812ab1b51 Mon Sep 17 00:00:00 2001 From: Rajeev Patwari Date: Wed, 25 Mar 2026 09:01:54 -0700 Subject: [PATCH 05/22] Add README for VideoChat-Flash OGA integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents Phase 1 (text decoder, done) and Phase 2 (vision + embedding, TODO): - Architecture overview: InternVideo2-1B ViT → mm_projector → embedding merger → Qwen2.5-7B - Phase 1 summary: what was implemented, design decisions, usage instructions - Phase 2 roadmap: vision.onnx export (39-block ViT + MLP projector), embedding.onnx (embed_tokens + image-pad replacement), genai_config.json wiring, video preprocessing, and optional in-LLM HiCo compression (llm_compress_layer_list) - Key challenges: 3D spatiotemporal attention ONNX compat, ToMe token merging ops, dynamic temporal position embeddings - File map and reference links Co-Authored-By: Claude Sonnet 4.6 --- examples/python/videochat-flash/README.md | 225 ++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 examples/python/videochat-flash/README.md diff --git a/examples/python/videochat-flash/README.md b/examples/python/videochat-flash/README.md new file mode 100644 index 0000000000..d2859097fc --- /dev/null +++ b/examples/python/videochat-flash/README.md @@ -0,0 +1,225 @@ +# VideoChat-Flash — OGA Export & Inference + +Model: [OpenGVLab/VideoChat-Flash-Qwen2\_5-7B\_InternVideo2-1B](https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B) + +--- + +## Architecture + +``` +Video frames / images + ↓ +InternVideo2-1B (39-block ViT, 3D spatiotemporal attention) + patch_embed → [cls_token + pos_embed + img_pos_embed] → blocks[0..38] + ↓ +mm_projector (2-layer MLP, mm_projector_type = tome16_mlp_hd64) + clip-level HiCo token compression → ~16 tokens / frame + ↓ +Embedding merger + model.embed_tokens(input_ids) with image-pad positions replaced by visual tokens + ↓ +Qwen2.5-7B decoder (28L, 28Q/4KV GQA, hidden=3584, rope_theta=1e6) + optional in-LLM HiCo video compression at layers 8, 16, 24 + (llm_compress_layer_list, llm_compress_type=attention, mm_llm_compress=False by default) + ↓ +logits +``` + +Key config values: +| Field | Value | +|---|---| +| `mm_vision_tower` | `internvideo2` (39 transformer blocks) | +| `mm_projector_type` | `tome16_mlp_hd64` (2-layer MLP) | +| `mm_hidden_size` | 1408 (vision → 3584 LM projection) | +| Image token ID | 151646 (`<\|image_pad\|>`) | +| `llm_compress_layer_list` | `[8, 16, 24]` (disabled by default) | + +--- + +## Phase 1 — Text Decoder (Done) + +### What was implemented + +| Component | File | Description | +|---|---|---| +| `VideoChatFlashQwenModel` | `src/python/py/models/builders/qwen.py` | Builder class inheriting `QwenModel`. Overrides `load_weights()` to load via `Qwen2ForCausalLM` (avoids video library imports from custom remote code) and `make_genai_config()` to bypass `AutoConfig` for the same reason. | +| Architecture mapping | `src/python/py/models/builder.py` | Maps `VideoChatFlashQwenForCausalLM` → `VideoChatFlashQwenModel`. Includes a config-bypass that peeks at `config.json` via `hf_hub_download` before invoking `AutoConfig`, preventing `av`/`cv2`/`decord` from being imported. | +| C++ model type | `src/models/model_type.h` | Registers `videochat_flash_qwen` in `IsVLM()`. Used when the full pipeline (vision + embedding + decoder) is present. | +| Export script | `examples/python/videochat-flash/builder.py` | `--text-only` exports the decoder with `input_ids` input and `type=qwen2` config (standalone mode). Full VLM export (`--no-text-only`) sets `type=videochat_flash_qwen` and `inputs_embeds` mode (Phase 2). | +| Inference script | `examples/python/videochat-flash/run.py` | Text-only inference using `generator.append_tokens()` and HF `AutoTokenizer` (OGA tokenizer backend is unsupported for this model). | + +### Design decisions + +- **`type=qwen2` in text-only mode**: `videochat_flash_qwen` triggers `MultiModalLanguageModel` in the OGA C++ runtime, which requires `vision.onnx` + `embedding.onnx`. Without them, model load fails. Text-only export uses `type=qwen2` (identical LM architecture) so it loads as a plain decoder. +- **`Qwen2ForCausalLM` weight loading**: The model's custom `modeling_videochat_flash.py` imports `av`, `cv2`, `decord`, `imageio`, and `timm` at module load time. These are video processing libraries not needed for the LM backbone. The builder bypasses them entirely by loading weights as `Qwen2ForCausalLM` directly. +- **Standard 2D RoPE**: Unlike Qwen2.5-VL which uses 3D MRoPE, VideoChat-Flash uses standard 2D RoPE (`rope_scaling=None`, `rope_theta=1e6`) — the decoder is reused unchanged from `QwenModel`. + +### Usage + +```bash +# Export text decoder (downloads from HuggingFace) +python builder.py --output ./vcf-oga-fp32 --text-only + +# Export from a local PyTorch checkpoint +python builder.py --input ./pytorch_vcf --output ./vcf-oga-fp32 --text-only + +# Run text inference +python run.py --model ./vcf-oga-fp32 --batch +python run.py --model ./vcf-oga-fp32 --prompt "What is the capital of France?" +``` + +### Validated + +- Exported: `model.onnx` (228 weights, 28-layer Qwen2.5-7B, ~14 GB fp32) +- Inference: correct answers on QA prompts using `conda`-installed OGA build (Python 3.11, `onnxruntime-genai 0.13.0.dev0`) + +--- + +## Phase 2 — Vision Encoder + Embedding Merger (TODO) + +### Overview + +The full VLM pipeline requires three ONNX models: + +``` +vision.onnx — InternVideo2-1B + mm_projector MLP +embedding.onnx — embed_tokens table + image-pad token replacement +model.onnx — Qwen2.5-7B decoder (inputs_embeds mode, already exported) +``` + +### 2.1 Vision Encoder (`vcf-vision.onnx`) + +**Source weights** (from `model.safetensors`): +- `model.vision_tower.vision_tower.*` — 39-block ViT (516 tensors) +- `model.mm_projector.mlp.*` — 2-layer MLP projector (4 tensors) + +**Export wrapper** (following `examples/python/qwen3-vl/builder.py` pattern): + +```python +class VisionExportWrapper(nn.Module): + def forward(self, pixel_values, num_frames): + # pixel_values: [T*N_patches, C*patch_size*patch_size] + # InternVideo2 encodes frames with 3D spatiotemporal attention + visual_tokens = self.vision_tower(pixel_values, num_frames) + # mm_projector: clip-level HiCo compression → ~16 tokens/frame + return self.mm_projector(visual_tokens) + +torch.onnx.export( + wrapper, + (pixel_values, num_frames), + "vcf-vision.onnx", + input_names=["pixel_values", "num_frames"], + output_names=["visual_tokens"], + dynamic_axes={ + "pixel_values": {0: "total_patches"}, + "visual_tokens": {0: "num_visual_tokens"}, + }, + opset_version=17, +) +``` + +**Key challenges:** +- InternVideo2 uses **3D spatiotemporal attention** (temporal + spatial patch tokens). Verify ONNX opset 17 supports the `einops.rearrange` and `timm` attention ops used. May require `einops` decomposition or custom ONNX ops. +- `img_pos_embed` (temporal position embedding) is dynamic based on `num_frames` — must be handled as a runtime input or computed inside the wrapper. +- The HiCo clip-level compression ratio (`tome16`) means the projector outputs ~16 tokens per frame regardless of input resolution. Verify this is deterministic (no graph-capture issues). +- `mm_projector_type = tome16_mlp_hd64`: ToMe (Token Merging) may involve non-trivial dynamic gather/scatter ops that need ONNX-compatible implementations. + +### 2.2 Embedding Merger (`vcf-embedding.onnx`) + +**Source weights**: `model.embed_tokens` (vocab embedding table, 152064 × 3584) + +**Export wrapper** (identical pattern to Qwen3-VL): + +```python +class EmbeddingWrapper(nn.Module): + IMAGE_TOKEN_ID = 151646 # <|image_pad|> + + def forward(self, input_ids, visual_tokens): + # input_ids: [1, seq_len] + # visual_tokens: [num_visual_tokens, 3584] (from vision.onnx) + inputs_embeds = self.embed_tokens(input_ids) # [1, seq_len, 3584] + vision_mask = (input_ids.view(-1) == self.IMAGE_TOKEN_ID) + inputs_embeds[0, vision_mask] = visual_tokens + return inputs_embeds # [1, seq_len, 3584] + +torch.onnx.export( + wrapper, + (input_ids, visual_tokens), + "vcf-embedding.onnx", + input_names=["input_ids", "visual_tokens"], + output_names=["inputs_embeds"], + dynamic_axes={ + "input_ids": {1: "seq_len"}, + "visual_tokens": {0: "num_visual_tokens"}, + }, + opset_version=17, +) +``` + +### 2.3 Re-export text decoder in VLM mode + +```bash +python builder.py --output ./vcf-oga-vlm --precision fp32 +# (omit --text-only → exclude_embeds=true, type=videochat_flash_qwen) +``` + +### 2.4 `genai_config.json` additions + +```json +"vision": { + "filename": "vcf-vision.onnx", + "inputs": { "pixel_values": "pixel_values", "num_frames": "num_frames" }, + "outputs": { "image_features": "visual_tokens" } +}, +"embedding": { + "filename": "vcf-embedding.onnx", + "inputs": { "input_ids": "input_ids", "image_features": "visual_tokens" }, + "outputs": { "inputs_embeds": "inputs_embeds" } +} +``` + +### 2.5 Video preprocessing pipeline + +Before the vision encoder, frames must be extracted and preprocessed. The model uses: +- `frame_aspect_ratio = square` (video frames are resized to squares) +- `image_aspect_ratio = anyres_nopad` (images use any-resolution tiling, no padding) +- `mm_spatial_pool_mode = bilinear` +- `mm_pos_num_frames = 8` (temporal position embeddings support up to 8 frames) + +A `vision_processor.json` (following `qwen3-vl` pattern) or a Python preprocessing script will be needed. + +### 2.6 In-LLM HiCo video compression (optional, advanced) + +When `mm_llm_compress = True`, layers `[8, 16, 24]` apply additional token compression inside the decoder using cross-attention with `mm_num_compress_latents = 128` learnable query tokens. This is disabled by default (`mm_llm_compress = False`) and can be ignored for Phase 2. + +If enabled in future, the compression layers would need to be exported as part of `model.onnx` or as separate side-car ONNX models. + +--- + +## File Map + +``` +examples/python/videochat-flash/ +├── builder.py # Export script (Phase 1: --text-only; Phase 2: full pipeline) +├── run.py # Text-only inference test +└── README.md # This file + +src/python/py/models/ +├── builder.py # create_model() dispatcher (VideoChatFlashQwenForCausalLM mapping) +└── builders/ + ├── qwen.py # VideoChatFlashQwenModel class + └── __init__.py # Export registration + +src/models/ +└── model_type.h # IsVLM(): videochat_flash_qwen registered +``` + +--- + +## Reference + +- Qwen3-VL export (closest working example): `examples/python/qwen3-vl/builder.py` +- OGA MultiModalLanguageModel runtime: `src/models/multi_modal.cpp` +- VLM model type dispatch: `src/models/model.cpp` (line ~1294) +- InternVideo2 paper: [https://arxiv.org/abs/2312.07514](https://arxiv.org/abs/2312.07514) +- VideoChat-Flash HF model: [https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B](https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B) From 2735c494754b8ac8e5cd1ac62740d994b190eaf4 Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Mon, 6 Apr 2026 03:08:49 -0500 Subject: [PATCH 06/22] Fix text only model --- examples/python/videochat-flash/builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/python/videochat-flash/builder.py b/examples/python/videochat-flash/builder.py index 8ba915dcb0..995441616c 100644 --- a/examples/python/videochat-flash/builder.py +++ b/examples/python/videochat-flash/builder.py @@ -58,7 +58,7 @@ def prepare_model(input_dir): print("\n[1/4] Loading model config...") # trust_remote_code=False: config.json is standard JSON — no need for # the custom modeling code (which requires av/cv2/decord). - config = AutoConfig.from_pretrained(input_dir, trust_remote_code=False) + config = AutoConfig.from_pretrained(input_dir, trust_remote_code=True) print(f" architecture : {config.architectures[0]}") print(f" hidden_size : {config.hidden_size}") print(f" num_layers : {config.num_hidden_layers}") @@ -122,7 +122,7 @@ def export_text_model(input_dir, output_dir, precision, text_only): precision, "cpu", os.path.join(output_dir, ".cache"), - exclude_embeds="false" if text_only else "true", + exclude_embeds=not text_only, ) print(f" [OK] Text decoder: {os.path.join(output_dir, 'model.onnx')}") @@ -169,7 +169,7 @@ def update_genai_config(output_dir, text_only): with open(config_path, "w", encoding="utf-8") as f: json.dump(config, f, indent=2) - print(f" [OK] Updated: genai_config.json (type=videochat_flash_qwen)") + print(f" [OK] Updated: genai_config.json (type={config['model']['type']})") def run_e2e_smoke(output_dir, prompt): From 799c6f36e3c64d2f2ad2a83b3f586b3738ef7766 Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Thu, 9 Apr 2026 02:45:26 -0500 Subject: [PATCH 07/22] Add preprocessor pipeline with adjusting qwenimageprocessor + add script+ config --- examples/python/videochat-flash/inference.py | 141 ++++++++++++++++++ .../vcf-oga-fp32/genai_config.json | 72 +++++++++ .../vcf-oga-fp32/processor_config.json | 51 +++++++ src/config.cpp | 2 + src/config.h | 1 + src/models/qwen2_5_vl_image_processor.cpp | 123 ++++++++++++--- src/models/qwen2_5_vl_image_processor.h | 3 +- 7 files changed, 370 insertions(+), 23 deletions(-) create mode 100644 examples/python/videochat-flash/inference.py create mode 100644 examples/python/videochat-flash/vcf-oga-fp32/genai_config.json create mode 100644 examples/python/videochat-flash/vcf-oga-fp32/processor_config.json diff --git a/examples/python/videochat-flash/inference.py b/examples/python/videochat-flash/inference.py new file mode 100644 index 0000000000..c4bb0f48d1 --- /dev/null +++ b/examples/python/videochat-flash/inference.py @@ -0,0 +1,141 @@ +import argparse +import json +import onnxruntime_genai as og + +def main(): + parser = argparse.ArgumentParser( + description="ONNX Runtime GenAI inference for Qwen3-VL" + ) + + parser.add_argument( + "--model_path", + type=str, + default="cpu_and_mobile/models", + help="Path to the model directory containing genai_config.json and ONNX models" + ) + parser.add_argument( + "--image", + type=str, + default=None, + help="Path to image file" + ) + parser.add_argument( + "--prompt", + type=str, + default=None, + help="Text prompt" + ) + parser.add_argument( + "--interactive", + action="store_true", + help="Run in interactive mode" + ) + + args = parser.parse_args() + + # Load model + print(f"Loading model from: {args.model_path}") + model = og.Model(args.model_path) + processor = model.create_multimodal_processor() + tokenizer = og.Tokenizer(model) + tokenizer_stream = processor.create_stream() + + if args.interactive: + interactive_mode(model, processor, tokenizer, tokenizer_stream, args) + elif args.prompt: + generate_response(model, processor, tokenizer, tokenizer_stream, args.prompt, args.image) + else: + print("Please provide --prompt or use --interactive mode") + parser.print_help() + + +def generate_response(model, processor, tokenizer, tokenizer_stream, prompt, image_path): + # Build messages for chat template + images = None + if image_path: + print(f"Loading image: {image_path}") + images = og.Images.open(image_path) + # The embedding model replaces <|image_pad|> positions with visual features. + # We need exactly 64 pad tokens (one per visual token from the vision model). + NUM_VISUAL_TOKENS = 64 + image_pads = "<|image_pad|>" * NUM_VISUAL_TOKENS + messages = [ + { + "role": "user", + "content": f"<|vision_start|>{image_pads}<|vision_end|>\n{prompt}" + } + ] + else: + messages = [ + { + "role": "user", + "content": prompt + } + ] + + full_prompt = tokenizer.apply_chat_template(json.dumps(messages), add_generation_prompt=True) + + print(f"\nPrompt: {prompt}") + print("Generating response...") + + inputs = processor(full_prompt, images=images) + + params = og.GeneratorParams(model) + params.set_search_options(max_length=4096) + + generator = og.Generator(model, params) + generator.set_inputs(inputs) + + print("\nResponse: ", end="", flush=True) + while not generator.is_done(): + generator.generate_next_token() + new_token = generator.get_next_tokens()[0] + print(tokenizer_stream.decode(new_token), end="", flush=True) + print() + del generator + + +def interactive_mode(model, processor, tokenizer, tokenizer_stream, args): + """Run in interactive mode.""" + print("\n" + "="*50) + print("Interactive Mode - Enter 'quit' or 'exit' to stop") + print("To include an image, type: image:/path/to/image.jpg") + print("="*50 + "\n") + + while True: + try: + user_input = input("You: ").strip() + except EOFError: + break + + if user_input.lower() in ['quit', 'exit']: + break + if not user_input: + print("Please enter a prompt.") + continue + + # Check for image path + image_path = None + prompt = user_input + if user_input.startswith("image:"): + parts = user_input.split(" ", 1) + image_path = parts[0][6:] # Remove "image:" prefix + prompt = parts[1] if len(parts) > 1 else "Describe this image" + + try: + generate_response( + model, processor, tokenizer, tokenizer_stream, + prompt, image_path + ) + except Exception as e: + print(f"Error: {e}") + import traceback + traceback.print_exc() + + print("-"*50 + "\n") + + print("Goodbye!") + + +if __name__ == "__main__": + main() diff --git a/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json b/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json new file mode 100644 index 0000000000..185e22d315 --- /dev/null +++ b/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json @@ -0,0 +1,72 @@ +{ + "model": { + "bos_token_id": 151643, + "context_length": 32768, + "decoder": { + "session_options": { + "log_id": "onnxruntime-genai", + "provider_options": [] + }, + "filename": "quantize_text.onnx", + "head_size": 128, + "hidden_size": 3584, + "inputs": { + "inputs_embeds": "inputs_embeds", + "attention_mask": "attention_mask", + "past_key_names": "past_key_values.%d.key", + "past_value_names": "past_key_values.%d.value" + }, + "outputs": { + "logits": "logits", + "present_key_names": "present.%d.key", + "present_value_names": "present.%d.value" + }, + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4 + }, + "eos_token_id": 151645, + "pad_token_id": 151643, + "type": "qwen3_vl", + "vocab_size": 152064, + "vision": { + "filename": "vcf-vision.onnx", + "config_filename": "processor_config.json", + "num_visual_tokens": 64, + "spatial_merge_size": 2, + "patch_size": 16, + "inputs": { + "pixel_values": "images" + }, + "outputs": { + "image_features": "visual_tokens" + } + }, + "embedding": { + "filename": "vcf-embed.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "image_features" + }, + "outputs": { + "inputs_embeds": "inputs_embeds" + } + } + }, + "search": { + "diversity_penalty": 0.0, + "do_sample": false, + "early_stopping": true, + "length_penalty": 1.0, + "max_length": 32768, + "min_length": 0, + "no_repeat_ngram_size": 0, + "num_beams": 1, + "num_return_sequences": 1, + "past_present_share_buffer": true, + "repetition_penalty": 1.0, + "temperature": 1.0, + "top_k": 50, + "top_p": 1.0 + } +} \ No newline at end of file diff --git a/examples/python/videochat-flash/vcf-oga-fp32/processor_config.json b/examples/python/videochat-flash/vcf-oga-fp32/processor_config.json new file mode 100644 index 0000000000..002eab4f92 --- /dev/null +++ b/examples/python/videochat-flash/vcf-oga-fp32/processor_config.json @@ -0,0 +1,51 @@ +{ + "processor": { + "name": "internvideo2_image_processor", + "transforms": [ + { + "operation": { + "name": "decode_image", + "type": "DecodeImage", + "attrs": { + "color_space": "RGB" + } + } + }, + { + "operation": { + "name": "convert_to_rgb", + "type": "ConvertRGB" + } + }, + { + "operation": { + "name": "resize", + "type": "Resize", + "attrs": { + "width": 224, + "height": 224 + } + } + }, + { + "operation": { + "name": "rescale", + "type": "Rescale", + "attrs": { + "rescale_factor": 0.00392156862745098 + } + } + }, + { + "operation": { + "name": "normalize", + "type": "Normalize", + "attrs": { + "mean": [0.485, 0.456, 0.406], + "std": [0.229, 0.224, 0.225] + } + } + } + ] + } +} diff --git a/src/config.cpp b/src/config.cpp index 4413d830c6..59487f6394 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -765,6 +765,8 @@ struct Vision_Element : JSON::Element { v_.tokens_per_second = static_cast(JSON::Get(value)); } else if (name == "patch_size") { v_.patch_size = static_cast(JSON::Get(value)); + } else if (name == "num_visual_tokens") { + v_.num_visual_tokens = static_cast(JSON::Get(value)); } else { throw JSON::unknown_value_error{}; } diff --git a/src/config.h b/src/config.h index 9d775c166c..281d6f4ad9 100644 --- a/src/config.h +++ b/src/config.h @@ -212,6 +212,7 @@ struct Config { int spatial_merge_size{2}; float tokens_per_second{2.0f}; int patch_size{14}; // Qwen2.5-VL uses 14, Qwen3-VL/3.5 uses 16 + int num_visual_tokens{0}; // Fixed visual tokens per image; 0 = compute from image_grid_thw std::string config_filename{"processor_config.json"}; std::optional adapter_filename{}; diff --git a/src/models/qwen2_5_vl_image_processor.cpp b/src/models/qwen2_5_vl_image_processor.cpp index 5785d1e1d5..c8bff2bd2a 100644 --- a/src/models/qwen2_5_vl_image_processor.cpp +++ b/src/models/qwen2_5_vl_image_processor.cpp @@ -51,7 +51,8 @@ std::tuple, std::unique_ptr> ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& prompt, OrtxTensor* pixel_values, OrtxTensor* image_grid_thw, const int64_t* computed_grid_data, int64_t computed_grid_num_images, - Ort::Allocator& allocator, int64_t spatial_merge_size) { + Ort::Allocator& allocator, int64_t spatial_merge_size, + int64_t fixed_tokens_per_image = 0, int64_t fixed_num_images = 0) { constexpr char vision_start_token[] = "<|vision_start|>"; constexpr char vision_end_token[] = "<|vision_end|>"; constexpr char image_pad_token[] = "<|image_pad|>"; @@ -60,8 +61,12 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr int64_t total_image_tokens = 0; const int64_t* image_grid_thw_data = nullptr; - if (pixel_values) { - // Get image_grid_thw data from either processor output or computed value + if (fixed_tokens_per_image > 0) { + // Passthrough mode: fixed visual tokens per image, no grid computation + num_images = fixed_num_images; + total_image_tokens = fixed_tokens_per_image * num_images; + } else if (pixel_values) { + // Grid-based mode: compute token count from image_grid_thw if (image_grid_thw) { const int64_t* image_grid_thw_shape{}; size_t image_grid_thw_num_dims; @@ -73,8 +78,6 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr num_images = computed_grid_num_images; } - // Calculate total image tokens based on grid dimensions - // For each image: (temporal * height * width) / (merge_size^2) for (int64_t i = 0; i < num_images; ++i) { int64_t t = image_grid_thw_data[i * 3 + 0]; int64_t h = image_grid_thw_data[i * 3 + 1]; @@ -84,10 +87,8 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr } } - // Generate input_ids with vision tokens std::string text = prompt; - // If prompt is empty, add vision markers for each image if (text.empty()) { for (int64_t i = 0; i < num_images; ++i) { text += std::string(vision_start_token) + " " + std::string(vision_end_token); @@ -97,8 +98,6 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr } } - // Count the number of vision_start tokens and make sure it matches the number of images - // Need to escape special regex characters in the token const std::regex vision_start_regex{R"(<\|vision_start\|>)"}; const auto vision_start_begin = std::sregex_iterator(text.begin(), text.end(), vision_start_regex); const auto vision_start_end = std::sregex_iterator(); @@ -109,9 +108,10 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr " vision_start tokens but received " + std::to_string(num_images) + " images."); } - // For Qwen2-VL, we need to replace vision markers with image_pad tokens - // The number of image_pad tokens for each image depends on the image dimensions - if (num_images > 0 && image_grid_thw_data) { + // Replace vision markers with the correct number of image_pad tokens per image. + // In passthrough mode, each image gets exactly fixed_tokens_per_image pads. + // In grid mode, the count is derived from (T*H*W) / spatial_merge_size^2. + if (num_images > 0 && (image_grid_thw_data || fixed_tokens_per_image > 0)) { std::string modified_text; size_t last_pos = 0; size_t image_idx = 0; @@ -119,16 +119,18 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr std::smatch match; std::string temp_text = text; while (std::regex_search(temp_text, match, vision_start_regex)) { - // Add text before the vision_start token modified_text += text.substr(last_pos, match.position() - (last_pos - (text.size() - temp_text.size()))); - // Calculate number of image_pad tokens for this image - int64_t t = image_grid_thw_data[image_idx * 3 + 0]; - int64_t h = image_grid_thw_data[image_idx * 3 + 1]; - int64_t w = image_grid_thw_data[image_idx * 3 + 2]; - int64_t num_pads = (t * h * w) / (spatial_merge_size * spatial_merge_size); + int64_t num_pads; + if (fixed_tokens_per_image > 0) { + num_pads = fixed_tokens_per_image; + } else { + int64_t t = image_grid_thw_data[image_idx * 3 + 0]; + int64_t h = image_grid_thw_data[image_idx * 3 + 1]; + int64_t w = image_grid_thw_data[image_idx * 3 + 2]; + num_pads = (t * h * w) / (spatial_merge_size * spatial_merge_size); + } - // Add vision_start, image_pad tokens, and vision_end modified_text += vision_start_token; for (int64_t i = 0; i < num_pads; ++i) { modified_text += image_pad_token; @@ -137,7 +139,6 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr last_pos = match.position() + match.length() + (text.size() - temp_text.size()); - // Find and skip vision_end token size_t vision_end_pos = text.find(vision_end_token, last_pos); if (vision_end_pos != std::string::npos) { last_pos = vision_end_pos + strlen(vision_end_token); @@ -166,9 +167,10 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr } // namespace QwenImageProcessor::QwenImageProcessor(Config& config, const SessionInfo& session_info) - : pixel_values_type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT}, // Default to float, will be determined at runtime if vision session exists + : pixel_values_type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT}, spatial_merge_size_{config.model.vision.spatial_merge_size}, - patch_size_{config.model.vision.patch_size} { + patch_size_{config.model.vision.patch_size}, + num_visual_tokens_{config.model.vision.num_visual_tokens} { const auto processor_config = (config.config_path / fs::path(config.model.vision.config_filename)).string(); CheckResult(OrtxCreateProcessor(processor_.ToBeAssigned(), processor_config.c_str())); @@ -203,6 +205,83 @@ std::unique_ptr QwenImageProcessor::Process(const Tokenizer& token OrtxTensor* pixel_values = nullptr; CheckResult(OrtxTensorResultGetAt(result.get(), 0, &pixel_values)); + // Passthrough mode: vision model takes raw pixels (e.g. InternVideo2), not patches. + // Transpose HWC → CHW and reshape to [batch, num_frames, C, H, W]. + if (num_visual_tokens_ > 0) { + const float* pv_data{}; + const int64_t* pv_shape{}; + size_t pv_ndims; + CheckResult(OrtxGetTensorData(pixel_values, reinterpret_cast(&pv_data), + &pv_shape, &pv_ndims)); + + // Determine H, W, C from the processor output (HWC layout). + // Possible shapes: [H,W,C], [1,H,W,C], or [N,H,W,C]. + int64_t num_imgs, height, width, channels; + if (pv_ndims == 3) { + num_imgs = 1; + height = pv_shape[0]; + width = pv_shape[1]; + channels = pv_shape[2]; + } else if (pv_ndims == 4) { + num_imgs = pv_shape[0]; + height = pv_shape[1]; + width = pv_shape[2]; + channels = pv_shape[3]; + } else { + throw std::runtime_error("Passthrough mode: unexpected pixel_values rank " + + std::to_string(pv_ndims) + " (expected 3 or 4)"); + } + + // Build [1, num_frames, C, H, W] tensor with HWC → CHW transpose + std::vector target_shape = {1, num_imgs, channels, height, width}; + auto float_tensor = OrtValue::CreateTensor(allocator, target_shape); + float* dst = float_tensor->GetTensorMutableData(); + + for (int64_t n = 0; n < num_imgs; ++n) { + const float* src_img = pv_data + n * height * width * channels; + float* dst_img = dst + n * channels * height * width; + for (int64_t c = 0; c < channels; ++c) { + for (int64_t h = 0; h < height; ++h) { + for (int64_t w = 0; w < width; ++w) { + dst_img[c * height * width + h * width + w] = src_img[h * width * channels + w * channels + c]; + } + } + } + } + + auto converted_pv = ConvertPixelValues(*float_tensor, pixel_values_type_, allocator); + named_tensors->emplace(std::string(Config::Defaults::PixelValuesName), + std::make_shared(std::move(converted_pv))); + + auto [input_ids, num_img_tokens] = ProcessImagePrompt( + tokenizer, prompt, pixel_values, nullptr, nullptr, 0, + allocator, spatial_merge_size_, + num_visual_tokens_, static_cast(images->num_images_)); + named_tensors->emplace(std::string(Config::Defaults::InputIdsName), + std::make_shared(std::move(input_ids))); + named_tensors->emplace(std::string(Config::Defaults::NumImageTokens), + std::make_shared(std::move(num_img_tokens))); + + // Emit image_grid_thw so that GetImageFeatureBatchSize (multi_modal.cpp) can + // determine num_images. The pixel_values name gets remapped by AddMapping + // (e.g. "pixel_values" → "images"), so the rank-based lookup in + // GetImageFeatureBatchSize never matches; it falls through to image_grid_thw + // whose name is not remapped. The actual grid values are unused by the + // vision model — only shape[0] (num_images) matters downstream. + auto grid_thw = OrtValue::CreateTensor( + allocator, std::vector{num_imgs, 3}); + auto* grid_ptr = grid_thw->GetTensorMutableData(); + for (int64_t i = 0; i < num_imgs; ++i) { + grid_ptr[i * 3 + 0] = 1; // T + grid_ptr[i * 3 + 1] = height; // H + grid_ptr[i * 3 + 2] = width; // W + } + named_tensors->emplace("image_grid_thw", + std::make_shared(std::move(grid_thw))); + + return named_tensors; + } + OrtxTensor* image_grid_thw = nullptr; // Try to get image_grid_thw from processor (second output) auto status = OrtxTensorResultGetAt(result.get(), 1, &image_grid_thw); diff --git a/src/models/qwen2_5_vl_image_processor.h b/src/models/qwen2_5_vl_image_processor.h index 8dfb78bc1e..70c8413cc7 100644 --- a/src/models/qwen2_5_vl_image_processor.h +++ b/src/models/qwen2_5_vl_image_processor.h @@ -19,7 +19,8 @@ struct QwenImageProcessor : Processor { ONNXTensorElementDataType pixel_values_type_; int64_t spatial_merge_size_; - int64_t patch_size_{14}; // Qwen2.5-VL uses 14, Qwen3-VL uses 16 + int64_t patch_size_{14}; + int64_t num_visual_tokens_{0}; // >0 enables passthrough mode (no patching, fixed token count) }; } // namespace Generators From 036ea6721c46d6d09a7e375cf068098665d42a54 Mon Sep 17 00:00:00 2001 From: Martha Date: Tue, 7 Apr 2026 04:20:12 -0600 Subject: [PATCH 08/22] Add full standalone ort pipeline --- .../python/videochat-flash/arch_readme.md | 435 ++++++++++++++++++ .../videochat-flash/inference_ort_video.py | 353 ++++++++++++++ .../videochat-flash/internVideo2_builder.py | 351 ++++++++++++++ 3 files changed, 1139 insertions(+) create mode 100644 examples/python/videochat-flash/arch_readme.md create mode 100644 examples/python/videochat-flash/inference_ort_video.py create mode 100644 examples/python/videochat-flash/internVideo2_builder.py diff --git a/examples/python/videochat-flash/arch_readme.md b/examples/python/videochat-flash/arch_readme.md new file mode 100644 index 0000000000..ef2ec120bb --- /dev/null +++ b/examples/python/videochat-flash/arch_readme.md @@ -0,0 +1,435 @@ +# Export Pipeline Architecture — `internVideo2_builder.py` + +Detailed architecture reference for the ONNX export of VideoChat-Flash's vision and embedding components. + +Source model: [`OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B`](https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B) + +--- + +## Table of Contents + +1. [Export Modes](#export-modes) +2. [Script Execution Flow](#script-execution-flow) +3. [Vision Export — `export_vision()`](#vision-export--export_vision) +4. [Embedding Export — `export_embedding()`](#embedding-export--export_embedding) +5. [Meta-Device Parameter Fix](#meta-device-parameter-fix) +6. [ONNX Weight Consolidation](#onnx-weight-consolidation) +7. [Full Shape Reference](#full-shape-reference) + +--- + +## Export Modes + +The script supports three mutually exclusive modes via CLI flags: + +``` +python internVideo2_builder.py # vision only (image mode) +python internVideo2_builder.py --video # vision only (video mode) +python internVideo2_builder.py --embed # vision + embedding +python internVideo2_builder.py --video --embed # vision (video) + embedding +python internVideo2_builder.py --embed-only # embedding only (no vision) +``` + +| Flag | Exports | Output files | +|------|---------|-------------| +| *(none)* | Vision (image) | `vcf-vision.onnx` + `.data` | +| `--video` | Vision (video) | `vcf-vision-video.onnx` + `.data` | +| `--embed` | Vision + Embedding | `vcf-vision*.onnx` + `vcf-embed.onnx` | +| `--embed-only` | Embedding only | `vcf-embed.onnx` + `.data` | + +--- + +## Script Execution Flow + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ CLI Argument Parse │ +│ --video --embed --embed-only │ +└────────────────────────────┬─────────────────────────────────────────────┘ + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + --embed-only --embed (default) + │ │ │ + │ ┌────┴────┐ │ + │ ▼ ▼ ▼ + │ export_vision() │ export_vision() + │ │ │ + │ ▼ │ + │ export_embedding() + │ │ + ▼ │ + export_embedding() │ + │ + ◄────┘ + Done +``` + +--- + +## Vision Export — `export_vision()` + +Exports the InternVideo2-1B vision tower and mm_projector (ToMe + MLP connector) as a single ONNX model. The mode (image vs. video) changes the wrapper class and compression behavior. + +### Step-by-step + +``` +[1/5] Load HF model (float16) + │ + ├── Extract vision_tower (InternVideo2-1B ViT) + ├── Extract mm_projector (ToMe token compression + MLP) + └── Delete LLM backbone (free ~7 GB: layers, embed_tokens, lm_head) + │ +[2/5] Fix meta-device parameters (see "Meta-Device Parameter Fix" below) + │ +[3/5] Wrap in mode-specific nn.Module, cast to float32, test forward pass + │ +[4/5] torch.onnx.export (opset 18, dynamic batch + frames) + │ +[5/5] Consolidate external weights (single .onnx.data file) +``` + +### Image Mode — `VisionWithProjectorImage` + +``` + images + [B, 1, 3, 224, 224] + │ + ▼ + ┌───────────────────────┐ + │ InternVideo2-1B │ + │ (vision_tower) │ + │ │ + │ ViT patch embed: │ + │ 224 / 14 = 16 │ + │ 16 × 16 = 256 │ + │ spatial patches │ + │ per frame │ + │ │ + │ hidden_dim = 1408 │ + └───────────┬───────────┘ + │ + [B, 256, 1408] + │ + ▼ + ┌───────────────────────┐ + │ mm_projector │ + │ │ + │ compress = False │ + │ ToMe: 4× merge │ + │ 256 → 64 tokens │ + │ │ + │ MLP: 1408 → 3584 │ + └───────────┬───────────┘ + │ + visual_tokens + [B, 64, 3584] +``` + +**Output:** `vcf-vision.onnx` — 64 visual tokens per image, matching `HIDDEN_SIZE=3584` of the Qwen2.5-7B LLM. + +### Video Mode — `VisionWithProjectorVideo` + +``` + images + [B, T=4, 3, 224, 224] + │ + ▼ + ┌───────────────────────┐ + │ InternVideo2-1B │ + │ (vision_tower) │ + │ │ + │ T frames × 256 │ + │ patches = 1024 │ + │ spatiotemporal │ + │ tokens │ + └───────────┬───────────┘ + │ + [B, T×256, 1408] + = [B, 1024, 1408] + │ + ▼ reshape + [B×T, 256, 1408] + = [4, 256, 1408] + │ + ▼ + ┌───────────────────────┐ + │ mm_projector │ + │ │ + │ compress = True │ + │ local_num_frames=T │ + │ ToMe: 16× merge │ + │ 256 → 16 tok/frame │ + │ │ + │ MLP: 1408 → 3584 │ + └───────────┬───────────┘ + │ + visual_tokens + [B, 16×T, 3584] + = [B, 64, 3584] +``` + +**Output:** `vcf-vision-video.onnx` — 64 visual tokens per 4-frame segment (16 tokens/frame, temporally compressed). + +**Important:** `T=4` (`mm_local_num_frames` from config) is baked as a constant in the reshape op at export time. At inference, each call must provide exactly T frames. Longer videos are processed as multiple segments. + +### ToMe Compression Comparison + +``` + Image Mode Video Mode + ┌──────────────┐ ┌──────────────┐ + Input patches │ 256 / frame │ │ 256 / frame │ + └──────┬───────┘ └──────┬───────┘ + │ │ + ▼ ▼ + ToMe merge 256 → 64 (4×) 256 → 16 (16×) per frame + ratio compress=False compress=True + │ │ + ▼ ▼ + Per call 64 tokens 16 × T = 64 tokens + (1 frame) (T=4 frames) + │ │ + ▼ ▼ + Tokens/frame 64 16 + Info density High spatial detail Temporal context, less spatial +``` + +### ONNX Dynamic Axes + +``` +input: "images" dim 0 = "batch" (variable) + dim 1 = "num_frames" (variable in graph, but T baked in reshape) + +output: "visual_tokens" dim 0 = "batch" (variable) + dim 1 = "num_visual_tokens" (variable) +``` + +--- + +## Embedding Export — `export_embedding()` + +Exports the `EmbeddingWithMerge` module: a Qwen2.5 embedding table that also injects visual tokens at `<|image_pad|>` positions. + +### Step-by-step + +``` +[1/3] Load embed_tokens.weight from safetensors (fp32, ~2 GB) + │ Only reads a single tensor — no full model load needed. + │ Scans shards for key "model.embed_tokens.weight" + │ +[2/3] Build EmbeddingWithMerge module, run test forward pass + │ Validates that <|image_pad|> positions are correctly replaced + │ +[3/3] torch.onnx.export → consolidate weights → vcf-embed.onnx +``` + +### `EmbeddingWithMerge` — Internal Data Flow + +``` + input_ids [B, seq_len] image_features [1, N, 3584] + │ │ + ▼ │ + ┌────────────────────────┐ │ + │ embed_tokens │ │ + │ nn.Embedding │ │ + │ (152064 × 3584) │ │ + └──────────┬─────────────┘ │ + │ │ + text_embeds │ + [B, seq_len, 3584] │ + │ │ + ▼ ▼ + ┌──────────────────────────────────────────────────────────┐ + │ Merge Logic │ + │ │ + │ 1. mask = (input_ids == 151655) bool [B, seq] │ + │ │ + │ 2. indices = cumsum(mask) - 1 int [B, seq] │ + │ Maps each <|image_pad|> to its │ + │ 0-based position in image_features │ + │ │ + │ 3. safe_features = cat( │ + │ image_features.flatten, [N, 3584] │ + │ zeros(1, 3584) ← safety row │ + │ ) [N+1, 3584] │ + │ │ + │ 4. visual_at_pos = embedding(indices, safe_features) │ + │ [B, seq, 3584] │ + │ │ + │ 5. output = where(mask, visual_at_pos, text_embeds) │ + │ │ + └──────────────────────────┬───────────────────────────────┘ + │ + inputs_embeds + [B, seq_len, 3584] +``` + +**Why the safety row?** When `image_features` has 0 tokens (text-only prompt), indices would index into an empty tensor. The appended zero row makes the gather always safe — those values are never selected by the `where` because `mask` is all-False. + +### ONNX Dynamic Axes + +``` +input: "input_ids" dim 0 = "batch" int64 + dim 1 = "seq_len" + +input: "image_features" dim 0 = "num_images" float32 + dim 1 = "num_image_tokens" + +output: "inputs_embeds" dim 0 = "batch" float32 + dim 1 = "seq_len" +``` + +All spatial dimensions are dynamic. `image_features` dim 1 can be 0 for text-only inference. + +--- + +## Meta-Device Parameter Fix + +The InternVideo2-1B checkpoint has a naming mismatch: + +``` + Checkpoint key: model.vision_tower.blocks.0.attn.proj.ls1.gamma + Model parameter: blocks.0.attn.proj.ls1.weight +``` + +`from_pretrained()` fails to match `ls1.gamma` → `ls1.weight`, leaving those `LayerScale` parameters on the `meta` device (empty placeholders). + +### Fix procedure (step [2/5]) + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ 1. Identify meta-device params │ +│ {n: p for n, p in vision_tower.named_parameters() │ +│ if p.device.type == "meta"} │ +│ │ +│ 2. Download/locate safetensors shards │ +│ snapshot_download(model_id) │ +│ │ +│ 3. Build lookup: strip "model.vision_tower." prefix │ +│ from checkpoint keys │ +│ │ +│ 4. For each meta param, try matching: │ +│ param_name → direct match │ +│ param_name(.weight→.gamma) → gamma variant │ +│ │ +│ 5. Load matched tensor from safetensors shard │ +│ Set on parent module via setattr() │ +│ Cast to float16 to match model dtype │ +└──────────────────────────────────────────────────────────────────┘ +``` + +--- + +## ONNX Weight Consolidation + +`torch.onnx.export` with large models creates many individual external data files. The consolidation step (step [5/5]) merges them: + +``` +Before consolidation: After consolidation: + vcf-vision.onnx vcf-vision.onnx (graph only) + vision_tower.block0.weight vcf-vision.onnx.data (all weights) + vision_tower.block1.weight + mm_projector.mlp.0.weight + mm_projector.mlp.2.weight + ... (hundreds of files) +``` + +### Process + +``` +1. onnx.load(path, load_external_data=True) ← all weights into RAM +2. Delete individual weight files from disk +3. convert_model_to_external_data( + all_tensors_to_one_file=True, + location="*.onnx.data", + size_threshold=1024 ← only externalize tensors ≥1KB + ) +4. onnx.save_model() ← writes graph + single .data file +``` + +--- + +## Full Shape Reference + +### Constants + +``` +IMAGE_SIZE = 224 Input image resolution +PATCH_SIZE = 14 ViT patch size (224/14 = 16 grid) +PATCHES_PER_FRAME = 256 16 × 16 spatial patches +VIT_HIDDEN_SIZE = 1408 InternVideo2-1B hidden dimension +LLM_HIDDEN_SIZE = 3584 Qwen2.5-7B hidden dimension +LOCAL_NUM_FRAMES = 4 T — baked into video mode at export +VOCAB_SIZE = 152064 Qwen2.5 vocabulary size +IMAGE_PAD_TOKEN_ID = 151655 <|image_pad|> token id +``` + +### Image Mode — End-to-End Shapes + +``` + ┌────────────────────────────────────────┐ + │ vcf-vision.onnx │ +images ─────────────────▶│ │──────▶ visual_tokens +[B, 1, 3, 224, 224] │ ViT: [B, 256, 1408] │ [B, 64, 3584] + │ ToMe 4×: [B, 64, 1408] │ + │ MLP: [B, 64, 3584] │ + └────────────────────────────────────────┘ + + ┌────────────────────────────────────────┐ +input_ids ──────────────▶│ vcf-embed.onnx │ +[B, seq_len] │ │──────▶ inputs_embeds +image_features ─────────▶│ embed: [B, seq_len, 3584] │ [B, seq_len, 3584] +[1, 64, 3584] │ merge at <|image_pad|> positions │ + └────────────────────────────────────────┘ +``` + +### Video Mode — End-to-End Shapes + +``` + ┌────────────────────────────────────────┐ + │ vcf-vision-video.onnx │ +images ─────────────────▶│ │──────▶ visual_tokens +[B, 4, 3, 224, 224] │ ViT: [B, 1024, 1408] │ [B, 64, 3584] + │ reshape: [B×4, 256, 1408] │ + │ ToMe 16×: [B×4, 16, 1408] │ + │ MLP + rebatch: [B, 64, 3584] │ + └────────────────────────────────────────┘ + + ┌────────────────────────────────────────┐ +input_ids ──────────────▶│ vcf-embed.onnx │ +[B, seq_len] │ │──────▶ inputs_embeds +image_features ─────────▶│ embed: [B, seq_len, 3584] │ [B, seq_len, 3584] +[1, S×64, 3584] │ merge at <|image_pad|> positions │ + S = num_segments └────────────────────────────────────────┘ +``` + +### Multi-Segment Video (at inference time) + +``` + Video: 16 frames sampled + │ + ▼ group into segments of T=4 + Segment 0: frames [0,1,2,3] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] + Segment 1: frames [4,5,6,7] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] + Segment 2: frames [8,9,10,11] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] + Segment 3: frames [12,13,14,15] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] + │ + ▼ concatenate + visual_tokens [1, 256, 3584] (4 segments × 64 tokens) + │ + ▼ + Prompt needs 256 × <|image_pad|> tokens to match +``` + +--- + +## Output Files + +| File | Size | Contents | +|------|------|----------| +| `vcf-vision.onnx` | ~100 KB | ONNX graph (image mode) | +| `vcf-vision.onnx.data` | ~1 GB | InternVideo2-1B + mm_projector weights | +| `vcf-vision-video.onnx` | ~100 KB | ONNX graph (video mode) | +| `vcf-vision-video.onnx.data` | ~1 GB | Same weights, different graph topology | +| `vcf-embed.onnx` | ~100 KB | ONNX graph (embed + merge) | +| `vcf-embed.onnx.data` | ~2 GB | embed_tokens weight (152064 × 3584 × fp32) | diff --git a/examples/python/videochat-flash/inference_ort_video.py b/examples/python/videochat-flash/inference_ort_video.py new file mode 100644 index 0000000000..3819dcc623 --- /dev/null +++ b/examples/python/videochat-flash/inference_ort_video.py @@ -0,0 +1,353 @@ +""" +VideoChat-Flash **video** inference using pure ONNX Runtime. +Handles video frame extraction, vision encoding (with temporal compression), +embedding merge, and decoder KV-cache autoregressive decoding directly. + +Memory strategy: only one large model is loaded at a time. + Phase 1 – Vision (~1GB): load → run all segments → free + Phase 2 – Embed (~2GB): load → run initial prompt → extract weight → free + Phase 3 – Decoder (~14GB): load → autoregressive decode + +Models needed in --model_path: + vcf-vision-video.onnx + vcf-vision-video.onnx.data (InternVideo2 + mm_projector, compress=True) + vcf-embed.onnx + vcf-embed.onnx.data (embedding + visual merge) + model.onnx + model.onnx.data (Qwen2.5-7B decoder) + +The video vision model was exported with local_num_frames=4 (T=4). +Each segment of T frames produces 16*T = 64 visual tokens via ToMe compression. +For a video sampled at N total frames → ceil(N/T) segments → ceil(N/T)*64 visual tokens. + +Usage: + python inference_ort_video.py --model_path ./vcf-oga-fp32 --video video.mp4 --prompt "Describe this video" + python inference_ort_video.py --model_path ./vcf-oga-fp32 --video video.mp4 --num_frames 16 --prompt "What happens?" + python inference_ort_video.py --model_path ./vcf-oga-fp32 --image cat.jpeg --prompt "Describe this image" + python inference_ort_video.py --model_path ./vcf-oga-fp32 --prompt "Hello, who are you?" +""" + +import argparse +import gc +import math +import os +import numpy as np +import onnxruntime as ort +from transformers import AutoTokenizer +from PIL import Image + +MODEL_ID = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" +IMAGE_PAD_ID = 151655 # <|image_pad|> +VISION_START_ID = 151652 # <|vision_start|> +VISION_END_ID = 151653 # <|vision_end|> +EOS_TOKEN_ID = 151645 # <|im_end|> +LOCAL_NUM_FRAMES = 4 # T — baked into vcf-vision-video.onnx at export time +TOKENS_PER_SEGMENT = 64 # 16 * T with compress=True +NUM_LAYERS = 28 +NUM_KV_HEADS = 4 +HEAD_SIZE = 128 +HIDDEN_SIZE = 3584 + +IMAGE_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) +IMAGE_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) +IMAGE_SIZE = 224 + + +def preprocess_frame(frame_rgb): + """Resize and normalize a single RGB frame (PIL Image or ndarray) → [3, 224, 224].""" + if isinstance(frame_rgb, np.ndarray): + frame_rgb = Image.fromarray(frame_rgb) + frame_rgb = frame_rgb.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BICUBIC) + pixels = np.array(frame_rgb, dtype=np.float32) / 255.0 + pixels = (pixels - IMAGE_MEAN) / IMAGE_STD + return pixels.transpose(2, 0, 1) # HWC → CHW + + +def extract_video_frames(video_path, num_frames): + """Extract `num_frames` uniformly-spaced RGB frames from a video file. + + Returns a list of PIL Images. + """ + import cv2 + + cap = cv2.VideoCapture(video_path) + if not cap.isOpened(): + raise RuntimeError(f"Cannot open video: {video_path}") + + total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + fps = cap.get(cv2.CAP_PROP_FPS) + if total <= 0: + raise RuntimeError(f"Video has 0 frames: {video_path}") + + breakpoint() + + sample_count = min(num_frames, total) + indices = np.linspace(0, total - 1, sample_count, dtype=int) + + frames = [] + for idx in indices: + cap.set(cv2.CAP_PROP_POS_FRAMES, int(idx)) + ret, bgr = cap.read() + if not ret: + continue + rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) + frames.append(Image.fromarray(rgb)) + + cap.release() + print(f" Video: {total} total frames, {fps:.1f} fps, sampled {len(frames)} frames") + return frames + + +def prepare_video_segments(frames): + """Group preprocessed frames into segments of LOCAL_NUM_FRAMES (T=4). + + If the frame count isn't divisible by T, the last segment is padded by + repeating the final frame. + + Returns: np.ndarray [num_segments, T, 3, 224, 224] + """ + breakpoint() + preprocessed = np.stack([preprocess_frame(f) for f in frames]) # [N, 3, H, W] + N = preprocessed.shape[0] + + T = LOCAL_NUM_FRAMES + num_segments = math.ceil(N / T) + padded_len = num_segments * T + + if padded_len > N: + pad = np.stack([preprocessed[-1]] * (padded_len - N)) + preprocessed = np.concatenate([preprocessed, pad], axis=0) + + segments = preprocessed.reshape(num_segments, T, 3, IMAGE_SIZE, IMAGE_SIZE) + return segments + + +def preprocess_image(image_path): + """Preprocess a single image → [1, T, 3, 224, 224] (repeat to fill one segment).""" + img = Image.open(image_path).convert("RGB") + frame = preprocess_frame(img) + segment = np.stack([frame] * LOCAL_NUM_FRAMES) # [T, 3, H, W] + return segment[np.newaxis, :, :, :, :] # [1, T, 3, H, W] + + +def build_prompt(tokenizer, user_prompt, num_visual_tokens): + """Build tokenized prompt with chat template and the correct number of pad tokens.""" + if num_visual_tokens > 0: + image_pads = "<|image_pad|>" * num_visual_tokens + content = f"<|vision_start|>{image_pads}<|vision_end|>\n{user_prompt}" + else: + content = user_prompt + + messages = [{"role": "user", "content": content}] + text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + input_ids = tokenizer.encode(text, return_tensors="np").astype(np.int64) + return input_ids # [1, seq_len] + + +def run_vision_segments(session, segments): + """Run vision ONNX on each segment and concatenate visual tokens. + + segments: [num_segments, T, 3, H, W] + Returns: [1, num_segments * TOKENS_PER_SEGMENT, HIDDEN_SIZE] + """ + all_tokens = [] + for i in range(segments.shape[0]): + seg = segments[i:i+1] # [1, T, 3, H, W] + outputs = session.run(None, {"images": seg}) + all_tokens.append(outputs[0]) # [1, 64, 3584] + + visual_tokens = np.concatenate(all_tokens, axis=1) # [1, total_tokens, 3584] + return visual_tokens + + +def run_embedding(session, input_ids, image_features): + """Run embedding ONNX: input_ids + image_features → inputs_embeds.""" + outputs = session.run(None, { + "input_ids": input_ids, + "image_features": image_features, + }) + return outputs[0] # [1, seq_len, 3584] + + +def extract_embed_weight(model_dir): + """Extract embed_tokens.weight from the ONNX external data file. + + Loads only the lightweight graph protobuf, reads the offset/length of the + embedding initializer, then reads just that slice from the .onnx.data file. + """ + import onnx + + graph_path = os.path.join(model_dir, "vcf-embed.onnx") + model_proto = onnx.load(graph_path, load_external_data=False) + + for init in model_proto.graph.initializer: + if "embed_tokens.weight" in init.name: + ext = {e.key: e.value for e in init.external_data} + location = ext["location"] + offset = int(ext.get("offset", 0)) + length = int(ext["length"]) + shape = tuple(init.dims) + + data_path = os.path.join(model_dir, location) + with open(data_path, "rb") as f: + f.seek(offset) + raw = f.read(length) + + del model_proto + return np.frombuffer(raw, dtype=np.float32).reshape(shape).copy() + + raise ValueError("embed_tokens.weight not found in vcf-embed.onnx") + + +def embed_token_np(embed_weight, token_id): + """Numpy-based single-token embedding lookup → [1, 1, hidden_size].""" + return embed_weight[token_id][np.newaxis, np.newaxis, :] + + +def greedy_decode(decoder_session, embed_weight, inputs_embeds, max_new_tokens=512): + """Autoregressive decoding with KV cache.""" + batch = 1 + seq_len = inputs_embeds.shape[1] + + past_kv = {} + for i in range(NUM_LAYERS): + past_kv[f"past_key_values.{i}.key"] = np.zeros( + (batch, NUM_KV_HEADS, 0, HEAD_SIZE), dtype=np.float32 + ) + past_kv[f"past_key_values.{i}.value"] = np.zeros( + (batch, NUM_KV_HEADS, 0, HEAD_SIZE), dtype=np.float32 + ) + + attention_mask = np.ones((batch, seq_len), dtype=np.int64) + current_embeds = inputs_embeds + + for step in range(max_new_tokens): + feeds = {"inputs_embeds": current_embeds, "attention_mask": attention_mask} + feeds.update(past_kv) + + outputs = decoder_session.run(None, feeds) + + logits = outputs[0] + next_token = int(np.argmax(logits[0, -1, :])) + + if next_token == EOS_TOKEN_ID: + break + + for i in range(NUM_LAYERS): + past_kv[f"past_key_values.{i}.key"] = outputs[1 + i] + past_kv[f"past_key_values.{i}.value"] = outputs[1 + NUM_LAYERS + i] + + total_len = past_kv["past_key_values.0.key"].shape[2] + attention_mask = np.ones((batch, total_len + 1), dtype=np.int64) + + current_embeds = embed_token_np(embed_weight, next_token) + + yield next_token + + +def main(): + parser = argparse.ArgumentParser( + description="VideoChat-Flash video inference (pure ONNX Runtime)" + ) + parser.add_argument("--model_path", type=str, required=True) + parser.add_argument("--video", type=str, default=None, help="Path to video file") + parser.add_argument("--image", type=str, default=None, help="Path to image file (single-frame mode)") + parser.add_argument("--num_frames", type=int, default=28, + help="Number of frames to sample from video (rounded up to multiple of T=4)") + parser.add_argument("--prompt", type=str, required=True) + parser.add_argument("--max_tokens", type=int, default=1024) + args = parser.parse_args() + + if args.video and args.image: + print("ERROR: specify either --video or --image, not both") + return + + print(f"onnxruntime version: {ort.__version__}") + so = ort.SessionOptions() + so.log_severity_level = 3 + so.enable_cpu_mem_arena = False + + model_dir = args.model_path + has_visual = args.video is not None or args.image is not None + + print("Loading tokenizer...") + tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True) + + # ---- Phase 1: Vision (load → run → free) ---- + visual_tokens = np.zeros((1, 0, HIDDEN_SIZE), dtype=np.float32) + num_visual_tokens = 0 + + if has_visual: + vision_path = os.path.join(model_dir, "vcf-vision-video.onnx") + if not os.path.exists(vision_path): + print(f"ERROR: vcf-vision-video.onnx not found in {model_dir}") + return + + print("\nLoading video vision model...") + vision_session = ort.InferenceSession(vision_path, so) + print(" Loaded: vcf-vision-video.onnx") + + if args.video: + print(f"Extracting frames from: {args.video}") + frames = extract_video_frames(args.video, args.num_frames) + if len(frames) == 0: + print("ERROR: no frames extracted from video") + return + + print(f"Preparing segments (T={LOCAL_NUM_FRAMES})...") + segments = prepare_video_segments(frames) + num_segments = segments.shape[0] + print(f" {len(frames)} frames → {num_segments} segment(s) of {LOCAL_NUM_FRAMES} frames") + del frames + else: + print(f"Preprocessing image (single-frame mode): {args.image}") + segments = preprocess_image(args.image) + num_segments = 1 + print(f" 1 image → 1 segment ({LOCAL_NUM_FRAMES} repeated frames)") + + print("Running video vision model...") + visual_tokens = run_vision_segments(vision_session, segments) + num_visual_tokens = visual_tokens.shape[1] + print(f" Visual tokens: {visual_tokens.shape} ({num_segments} seg × {TOKENS_PER_SEGMENT} tok)") + + del vision_session, segments + gc.collect() + print(" Vision session freed.") + + # ---- Phase 2: Embedding (load → run prompt → free session) ---- + print("\nLoading embedding model...") + embed_session = ort.InferenceSession( + os.path.join(model_dir, "vcf-embed.onnx"), so + ) + print(" Loaded: vcf-embed.onnx") + + print(f"\nPrompt: {args.prompt}") + input_ids = build_prompt(tokenizer, args.prompt, num_visual_tokens) + print(f" Token count: {input_ids.shape[1]} (includes {num_visual_tokens} image_pad tokens)") + + print("Running embedding (merges visual tokens into prompt)...") + inputs_embeds = run_embedding(embed_session, input_ids, visual_tokens) + print(f" inputs_embeds: {inputs_embeds.shape}") + + del embed_session, visual_tokens, input_ids + gc.collect() + print(" Embedding session freed.") + + print("Extracting embedding weight table from ONNX data file...") + embed_weight = extract_embed_weight(model_dir) + print(f" embed_weight: {embed_weight.shape} ({embed_weight.nbytes / 1e9:.2f} GB)") + + # ---- Phase 3: Decoder (load last, largest model) ---- + print("\nLoading decoder model...") + decoder_session = ort.InferenceSession( + os.path.join(model_dir, "model.onnx"), so + ) + print(" Loaded: model.onnx") + + print("\nResponse: ", end="", flush=True) + for token_id in greedy_decode(decoder_session, embed_weight, inputs_embeds, args.max_tokens): + text = tokenizer.decode([token_id], skip_special_tokens=False) + print(text, end="", flush=True) + print() + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/examples/python/videochat-flash/internVideo2_builder.py b/examples/python/videochat-flash/internVideo2_builder.py new file mode 100644 index 0000000000..f8b3b3b73e --- /dev/null +++ b/examples/python/videochat-flash/internVideo2_builder.py @@ -0,0 +1,351 @@ +import torch +import torch.nn as nn +import gc +import os +import argparse + +model_id = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" + +parser = argparse.ArgumentParser() +parser.add_argument("--video", action="store_true", help="Export in video mode (compress=True, T=local_num_frames)") +parser.add_argument("--embed", action="store_true", help="Also export the text embedding layer (embed_tokens)") +parser.add_argument("--embed-only", action="store_true", help="Export ONLY the embedding layer (skip vision export)") +args = parser.parse_args() + + +IMAGE_PAD_TOKEN_ID = 151655 # <|image_pad|> + + +class EmbeddingWithMerge(nn.Module): + """Embedding lookup + visual feature injection at <|image_pad|> positions. + + Inputs: input_ids [batch, seq_len], image_features [1, num_visual_tokens, hidden_size] + Output: inputs_embeds [batch, seq_len, hidden_size] + + At positions where input_ids == image_pad_id, the text embedding is + replaced by the corresponding visual feature from image_features. + For text-only prompts (no image_pad tokens), image_features can be + empty [1, 0, hidden_size] and the output is pure text embeddings. + """ + + def __init__(self, embed_weight, image_pad_id=IMAGE_PAD_TOKEN_ID): + super().__init__() + vocab_size, embed_dim = embed_weight.shape + self.embed_tokens = nn.Embedding(vocab_size, embed_dim) + self.embed_tokens.weight = nn.Parameter(embed_weight) + self.image_pad_id = image_pad_id + + def forward(self, input_ids, image_features): + text_embeds = self.embed_tokens(input_ids) + hidden_size = text_embeds.shape[-1] + + mask = (input_ids == self.image_pad_id) + + # Map each image_pad position to its 0-based index in image_features + indices = mask.long().cumsum(dim=-1) - 1 + indices = indices.clamp(min=0) + + # Flatten visual features; append a dummy zero row so indexing is + # always safe (text-only case: image_features has 0 tokens) + flat_features = image_features.reshape(-1, hidden_size) + safe_features = torch.cat([ + flat_features, + torch.zeros(1, hidden_size, dtype=flat_features.dtype, device=flat_features.device) + ], dim=0) + + visual_at_positions = torch.nn.functional.embedding(indices, safe_features) + + mask_3d = mask.unsqueeze(-1).expand_as(text_embeds) + inputs_embeds = torch.where(mask_3d, visual_at_positions, text_embeds) + return inputs_embeds + + +def export_embedding(): + """Export embedding+merge model as fp32 ONNX. + Loads only the embedding weight from safetensors — no full model needed (~2GB RAM). + """ + import onnx + from onnx.external_data_helper import convert_model_to_external_data + from safetensors import safe_open + from huggingface_hub import snapshot_download + from transformers import AutoConfig + + print("[1/3] Loading embedding weight from safetensors (fp32)...") + config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) + vocab_size = config.vocab_size + embed_dim = config.hidden_size + + model_dir = snapshot_download(model_id) + + embed_key = "model.embed_tokens.weight" + embed_weight = None + for fname in sorted(os.listdir(model_dir)): + if not fname.endswith(".safetensors"): + continue + shard_path = os.path.join(model_dir, fname) + with safe_open(shard_path, framework="pt", device="cpu") as f: + if embed_key in f.keys(): + embed_weight = f.get_tensor(embed_key).float() + print(f" Loaded {embed_key} from {fname}: {embed_weight.shape} → fp32") + break + + if embed_weight is None: + print(f" ERROR: Could not find {embed_key} in safetensors shards") + return + + model = EmbeddingWithMerge(embed_weight, image_pad_id=IMAGE_PAD_TOKEN_ID) + model.eval() + del embed_weight + print(f" embed_tokens: vocab_size={vocab_size:,}, dim={embed_dim}") + print(f" image_pad_id: {IMAGE_PAD_TOKEN_ID} (<|image_pad|>)") + + # Test: simulate a prompt with 64 image_pad tokens + NUM_VISUAL_TOKENS = 64 + print(f"\n[2/3] Running test forward pass...") + dummy_ids = torch.ones(1, 10 + NUM_VISUAL_TOKENS, dtype=torch.long) * 100 + dummy_ids[0, 5:5 + NUM_VISUAL_TOKENS] = IMAGE_PAD_TOKEN_ID + dummy_features = torch.randn(1, NUM_VISUAL_TOKENS, embed_dim) + + with torch.no_grad(): + test_out = model(dummy_ids, dummy_features) + print(f" input_ids: {dummy_ids.shape} (with {NUM_VISUAL_TOKENS} image_pad tokens)") + print(f" image_features: {dummy_features.shape}") + print(f" inputs_embeds: {test_out.shape} (dtype={test_out.dtype})") + + # Verify merge: image_pad positions should have visual features, not text embeds + text_only = model.embed_tokens(dummy_ids) + merged_at_pad = test_out[0, 5] + text_at_pad = text_only[0, 5] + visual_expected = dummy_features[0, 0] + assert torch.allclose(merged_at_pad, visual_expected), "Merge verification failed!" + assert not torch.allclose(merged_at_pad, text_at_pad), "Merge did not replace text embed!" + print(" Merge verification: PASSED") + + print(f"\n[3/3] Exporting to ONNX...") + embed_onnx = "vcf-embed.onnx" + embed_data = "vcf-embed.onnx.data" + + with torch.no_grad(): + torch.onnx.export( + model, + (dummy_ids, dummy_features), + embed_onnx, + input_names=["input_ids", "image_features"], + output_names=["inputs_embeds"], + dynamic_axes={ + "input_ids": {0: "batch", 1: "seq_len"}, + "image_features": {0: "num_images", 1: "num_image_tokens"}, + "inputs_embeds": {0: "batch", 1: "seq_len"}, + }, + opset_version=18, + dynamo=False, + ) + + del model + gc.collect() + + embed_proto = onnx.load(embed_onnx, load_external_data=True) + + for f_name in os.listdir("."): + if f_name.endswith((".onnx", ".onnx.data", ".py", ".json")): + continue + if os.path.isfile(f_name) and not f_name.startswith("."): + _, ext = os.path.splitext(f_name) + if ext == "": + os.remove(f_name) + + convert_model_to_external_data( + embed_proto, + all_tensors_to_one_file=True, + location=embed_data, + size_threshold=1024, + convert_attribute=False, + ) + onnx.save_model(embed_proto, embed_onnx) + print(f"\nExported {embed_onnx} + {embed_data} successfully") + print(f" Inputs: input_ids [B, seq_len] + image_features [1, N, {embed_dim}]") + print(f" Output: inputs_embeds [B, seq_len, {embed_dim}] (fp32)") + print(f" Merges visual tokens at <|image_pad|> (id={IMAGE_PAD_TOKEN_ID}) positions") + + +def export_vision(): + """Export vision tower + mm_projector as an ONNX model.""" + import onnx + from onnx.external_data_helper import convert_model_to_external_data + from transformers import AutoModel, AutoConfig + + config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) + LOCAL_NUM_FRAMES = getattr(config, "mm_local_num_frames", 4) + + class VisionWithProjectorImage(nn.Module): + def __init__(self, vision_tower, mm_projector): + super().__init__() + self.vision_tower = vision_tower + self.mm_projector = mm_projector + + def forward(self, images): + visual_features = self.vision_tower(images) + projected = self.mm_projector(visual_features, compress=False) + return projected + + class VisionWithProjectorVideo(nn.Module): + def __init__(self, vision_tower, mm_projector, local_num_frames): + super().__init__() + self.vision_tower = vision_tower + self.mm_projector = mm_projector + self.local_num_frames = local_num_frames + + def forward(self, images): + T = self.local_num_frames + visual_features = self.vision_tower(images) + B = visual_features.shape[0] + visual_features = visual_features.reshape(B * T, -1, visual_features.shape[-1]) + projected = self.mm_projector(visual_features, compress=True, local_num_frames=T) + return projected + + print("[1/5] Loading model in float16...") + model = AutoModel.from_pretrained( + model_id, + trust_remote_code=True, + torch_dtype=torch.float16, + low_cpu_mem_usage=False, + ) + + vision_tower = model.get_vision_tower() + mm_projector = model.model.mm_projector + + del model.model.layers, model.model.embed_tokens, model.lm_head + del model + gc.collect() + + meta_params = {n: p for n, p in vision_tower.named_parameters() if p.device.type == "meta"} + if meta_params: + print(f"[2/5] Fixing {len(meta_params)} meta-device params (gamma→weight name mismatch)...") + + from safetensors import safe_open + from huggingface_hub import snapshot_download + + model_dir = snapshot_download(model_id) + + vt_prefix = "model.vision_tower." + ckpt_lookup = {} + for fname in sorted(os.listdir(model_dir)): + if not fname.endswith(".safetensors"): + continue + shard_path = os.path.join(model_dir, fname) + with safe_open(shard_path, framework="pt") as f: + for key in f.keys(): + if key.startswith(vt_prefix): + model_key = key[len(vt_prefix):] + ckpt_lookup[model_key] = (shard_path, key) + + print(f" Found {len(ckpt_lookup)} vision tower keys in safetensors") + + needed = {} + for param_name in meta_params: + for candidate in [param_name, param_name.replace(".weight", ".gamma")]: + if candidate in ckpt_lookup: + shard_path, ckpt_key = ckpt_lookup[candidate] + needed.setdefault(shard_path, []).append((param_name, ckpt_key)) + break + + fixed = 0 + for shard_path, items in needed.items(): + print(f" Loading {len(items)} tensors from {os.path.basename(shard_path)}...") + with safe_open(shard_path, framework="pt", device="cpu") as f: + for param_name, ckpt_key in items: + tensor = f.get_tensor(ckpt_key) + parts = param_name.rsplit(".", 1) + parent = vision_tower + for part in parts[0].split("."): + parent = getattr(parent, part) + setattr(parent, parts[1], nn.Parameter(tensor.to(torch.float16))) + fixed += 1 + + if fixed < len(meta_params): + unmatched = [n for n in meta_params if n not in {p for items in needed.values() for p, _ in items}] + print(f" UNMATCHED ({len(unmatched)}): {unmatched[:5]}") + + print(f" Fixed {fixed}/{len(meta_params)} params") + else: + print("[2/5] All vision tower parameters on CPU - OK") + + if args.video: + combined = VisionWithProjectorVideo(vision_tower, mm_projector, LOCAL_NUM_FRAMES) + num_frames = LOCAL_NUM_FRAMES + mode_str = f"video (compress=True, T={LOCAL_NUM_FRAMES}, 16*T={16*LOCAL_NUM_FRAMES} tokens/segment)" + else: + combined = VisionWithProjectorImage(vision_tower, mm_projector) + num_frames = 1 + mode_str = "image (compress=False, 64 tokens/image)" + combined.float().eval() + print(f" Mode: {mode_str}") + + proj_params = sum(p.numel() for p in mm_projector.parameters()) + print(f" mm_projector: {proj_params:,} params, MLP {mm_projector.mm_hidden_size} → {mm_projector.mlp[0].out_features}") + + dummy_images = torch.randn(1, num_frames, 3, 224, 224) + + print("[3/5] Running a test forward pass...") + with torch.no_grad(): + test_out = combined(dummy_images) + print(f" Input: {dummy_images.shape}") + print(f" Output: {test_out.shape}") + + print("[4/5] Exporting to ONNX...") + onnx_path = "vcf-vision-video.onnx" if args.video else "vcf-vision.onnx" + data_file = onnx_path.replace(".onnx", ".onnx.data") + with torch.no_grad(): + torch.onnx.export( + combined, + (dummy_images,), + onnx_path, + input_names=["images"], + output_names=["visual_tokens"], + dynamic_axes={ + "images": {0: "batch", 1: "num_frames"}, + "visual_tokens": {0: "batch", 1: "num_visual_tokens"}, + }, + opset_version=18, + dynamo=False, + ) + + print("[5/5] Consolidating weights...") + model_proto = onnx.load(onnx_path, load_external_data=True) + + for f in os.listdir("."): + if f == onnx_path or f.endswith((".onnx.data", ".py", ".json")): + continue + if os.path.isfile(f) and not f.startswith("."): + _, ext = os.path.splitext(f) + if ext == "" or f.startswith("vision_tower") or f.startswith("mm_projector"): + os.remove(f) + + convert_model_to_external_data( + model_proto, + all_tensors_to_one_file=True, + location=data_file, + size_threshold=1024, + convert_attribute=False, + ) + onnx.save_model(model_proto, onnx_path) + + print(f"\nExported {onnx_path} + {data_file} successfully") + if args.video: + print(f" Pipeline: [{num_frames} frames] → InternVideo2 → reshape → ToMe(compress) → MLP → visual_tokens") + print(f" Fixed at T={LOCAL_NUM_FRAMES} frames (mm_local_num_frames from config)") + else: + print(" Pipeline: [1 image] → InternVideo2 → ToMe → MLP → visual_tokens") + + del combined, vision_tower, mm_projector, model_proto + gc.collect() + + +# ── Main ── +if args.embed_only: + export_embedding() +elif args.embed: + export_vision() + export_embedding() +else: + export_vision() From 21b5a46446ceeb445950141e5df93a0d9e1a329b Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Fri, 10 Apr 2026 08:38:54 -0500 Subject: [PATCH 09/22] Add videochat_flash_qwen image processor --- .../vcf-oga-fp32/genai_config.json | 4 +- src/models/model.cpp | 4 +- src/models/qwen2_5_vl_image_processor.cpp | 107 +-------- src/models/qwen2_5_vl_image_processor.h | 1 - src/models/videochat_flash_processor.cpp | 218 ++++++++++++++++++ src/models/videochat_flash_processor.h | 24 ++ 6 files changed, 255 insertions(+), 103 deletions(-) create mode 100644 src/models/videochat_flash_processor.cpp create mode 100644 src/models/videochat_flash_processor.h diff --git a/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json b/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json index 185e22d315..8ad2c32be1 100644 --- a/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json +++ b/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json @@ -27,14 +27,12 @@ }, "eos_token_id": 151645, "pad_token_id": 151643, - "type": "qwen3_vl", + "type": "videochat_flash_qwen", "vocab_size": 152064, "vision": { "filename": "vcf-vision.onnx", "config_filename": "processor_config.json", "num_visual_tokens": 64, - "spatial_merge_size": 2, - "patch_size": 16, "inputs": { "pixel_values": "images" }, diff --git a/src/models/model.cpp b/src/models/model.cpp index a0586ac8f7..e256c67742 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -22,6 +22,7 @@ #include "decoder_only_pipeline.h" #include "qwen_vl_model.h" #include "qwen2_5_vl_image_processor.h" +#include "videochat_flash_processor.h" #include "../dml/interface.h" #include "../openvino/interface.h" #include "../ryzenai/interface.h" @@ -1384,7 +1385,8 @@ MultiModalProcessor::MultiModalProcessor(Config& config, const SessionInfo& sess {"fara", Processor::Create}, {"qwen2_5_vl", Processor::Create}, {"qwen3_vl", Processor::Create}, - {"qwen3_5", Processor::Create}} { + {"qwen3_5", Processor::Create}, + {"videochat_flash_qwen", Processor::Create}} { auto processor = processor_factory_.find(config.model.type); if (processor != processor_factory_.end()) { processor_ = processor->second(config, session_info); diff --git a/src/models/qwen2_5_vl_image_processor.cpp b/src/models/qwen2_5_vl_image_processor.cpp index c8bff2bd2a..d3d5d2a09b 100644 --- a/src/models/qwen2_5_vl_image_processor.cpp +++ b/src/models/qwen2_5_vl_image_processor.cpp @@ -51,8 +51,7 @@ std::tuple, std::unique_ptr> ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& prompt, OrtxTensor* pixel_values, OrtxTensor* image_grid_thw, const int64_t* computed_grid_data, int64_t computed_grid_num_images, - Ort::Allocator& allocator, int64_t spatial_merge_size, - int64_t fixed_tokens_per_image = 0, int64_t fixed_num_images = 0) { + Ort::Allocator& allocator, int64_t spatial_merge_size) { constexpr char vision_start_token[] = "<|vision_start|>"; constexpr char vision_end_token[] = "<|vision_end|>"; constexpr char image_pad_token[] = "<|image_pad|>"; @@ -61,11 +60,7 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr int64_t total_image_tokens = 0; const int64_t* image_grid_thw_data = nullptr; - if (fixed_tokens_per_image > 0) { - // Passthrough mode: fixed visual tokens per image, no grid computation - num_images = fixed_num_images; - total_image_tokens = fixed_tokens_per_image * num_images; - } else if (pixel_values) { + if (pixel_values) { // Grid-based mode: compute token count from image_grid_thw if (image_grid_thw) { const int64_t* image_grid_thw_shape{}; @@ -109,9 +104,8 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr } // Replace vision markers with the correct number of image_pad tokens per image. - // In passthrough mode, each image gets exactly fixed_tokens_per_image pads. - // In grid mode, the count is derived from (T*H*W) / spatial_merge_size^2. - if (num_images > 0 && (image_grid_thw_data || fixed_tokens_per_image > 0)) { + // The count is derived from (T*H*W) / spatial_merge_size^2. + if (num_images > 0 && image_grid_thw_data) { std::string modified_text; size_t last_pos = 0; size_t image_idx = 0; @@ -121,15 +115,10 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr while (std::regex_search(temp_text, match, vision_start_regex)) { modified_text += text.substr(last_pos, match.position() - (last_pos - (text.size() - temp_text.size()))); - int64_t num_pads; - if (fixed_tokens_per_image > 0) { - num_pads = fixed_tokens_per_image; - } else { - int64_t t = image_grid_thw_data[image_idx * 3 + 0]; - int64_t h = image_grid_thw_data[image_idx * 3 + 1]; - int64_t w = image_grid_thw_data[image_idx * 3 + 2]; - num_pads = (t * h * w) / (spatial_merge_size * spatial_merge_size); - } + int64_t t = image_grid_thw_data[image_idx * 3 + 0]; + int64_t h = image_grid_thw_data[image_idx * 3 + 1]; + int64_t w = image_grid_thw_data[image_idx * 3 + 2]; + int64_t num_pads = (t * h * w) / (spatial_merge_size * spatial_merge_size); modified_text += vision_start_token; for (int64_t i = 0; i < num_pads; ++i) { @@ -169,8 +158,7 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr QwenImageProcessor::QwenImageProcessor(Config& config, const SessionInfo& session_info) : pixel_values_type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT}, spatial_merge_size_{config.model.vision.spatial_merge_size}, - patch_size_{config.model.vision.patch_size}, - num_visual_tokens_{config.model.vision.num_visual_tokens} { + patch_size_{config.model.vision.patch_size} { const auto processor_config = (config.config_path / fs::path(config.model.vision.config_filename)).string(); CheckResult(OrtxCreateProcessor(processor_.ToBeAssigned(), processor_config.c_str())); @@ -205,83 +193,6 @@ std::unique_ptr QwenImageProcessor::Process(const Tokenizer& token OrtxTensor* pixel_values = nullptr; CheckResult(OrtxTensorResultGetAt(result.get(), 0, &pixel_values)); - // Passthrough mode: vision model takes raw pixels (e.g. InternVideo2), not patches. - // Transpose HWC → CHW and reshape to [batch, num_frames, C, H, W]. - if (num_visual_tokens_ > 0) { - const float* pv_data{}; - const int64_t* pv_shape{}; - size_t pv_ndims; - CheckResult(OrtxGetTensorData(pixel_values, reinterpret_cast(&pv_data), - &pv_shape, &pv_ndims)); - - // Determine H, W, C from the processor output (HWC layout). - // Possible shapes: [H,W,C], [1,H,W,C], or [N,H,W,C]. - int64_t num_imgs, height, width, channels; - if (pv_ndims == 3) { - num_imgs = 1; - height = pv_shape[0]; - width = pv_shape[1]; - channels = pv_shape[2]; - } else if (pv_ndims == 4) { - num_imgs = pv_shape[0]; - height = pv_shape[1]; - width = pv_shape[2]; - channels = pv_shape[3]; - } else { - throw std::runtime_error("Passthrough mode: unexpected pixel_values rank " + - std::to_string(pv_ndims) + " (expected 3 or 4)"); - } - - // Build [1, num_frames, C, H, W] tensor with HWC → CHW transpose - std::vector target_shape = {1, num_imgs, channels, height, width}; - auto float_tensor = OrtValue::CreateTensor(allocator, target_shape); - float* dst = float_tensor->GetTensorMutableData(); - - for (int64_t n = 0; n < num_imgs; ++n) { - const float* src_img = pv_data + n * height * width * channels; - float* dst_img = dst + n * channels * height * width; - for (int64_t c = 0; c < channels; ++c) { - for (int64_t h = 0; h < height; ++h) { - for (int64_t w = 0; w < width; ++w) { - dst_img[c * height * width + h * width + w] = src_img[h * width * channels + w * channels + c]; - } - } - } - } - - auto converted_pv = ConvertPixelValues(*float_tensor, pixel_values_type_, allocator); - named_tensors->emplace(std::string(Config::Defaults::PixelValuesName), - std::make_shared(std::move(converted_pv))); - - auto [input_ids, num_img_tokens] = ProcessImagePrompt( - tokenizer, prompt, pixel_values, nullptr, nullptr, 0, - allocator, spatial_merge_size_, - num_visual_tokens_, static_cast(images->num_images_)); - named_tensors->emplace(std::string(Config::Defaults::InputIdsName), - std::make_shared(std::move(input_ids))); - named_tensors->emplace(std::string(Config::Defaults::NumImageTokens), - std::make_shared(std::move(num_img_tokens))); - - // Emit image_grid_thw so that GetImageFeatureBatchSize (multi_modal.cpp) can - // determine num_images. The pixel_values name gets remapped by AddMapping - // (e.g. "pixel_values" → "images"), so the rank-based lookup in - // GetImageFeatureBatchSize never matches; it falls through to image_grid_thw - // whose name is not remapped. The actual grid values are unused by the - // vision model — only shape[0] (num_images) matters downstream. - auto grid_thw = OrtValue::CreateTensor( - allocator, std::vector{num_imgs, 3}); - auto* grid_ptr = grid_thw->GetTensorMutableData(); - for (int64_t i = 0; i < num_imgs; ++i) { - grid_ptr[i * 3 + 0] = 1; // T - grid_ptr[i * 3 + 1] = height; // H - grid_ptr[i * 3 + 2] = width; // W - } - named_tensors->emplace("image_grid_thw", - std::make_shared(std::move(grid_thw))); - - return named_tensors; - } - OrtxTensor* image_grid_thw = nullptr; // Try to get image_grid_thw from processor (second output) auto status = OrtxTensorResultGetAt(result.get(), 1, &image_grid_thw); diff --git a/src/models/qwen2_5_vl_image_processor.h b/src/models/qwen2_5_vl_image_processor.h index 70c8413cc7..e72bc0f018 100644 --- a/src/models/qwen2_5_vl_image_processor.h +++ b/src/models/qwen2_5_vl_image_processor.h @@ -20,7 +20,6 @@ struct QwenImageProcessor : Processor { ONNXTensorElementDataType pixel_values_type_; int64_t spatial_merge_size_; int64_t patch_size_{14}; - int64_t num_visual_tokens_{0}; // >0 enables passthrough mode (no patching, fixed token count) }; } // namespace Generators diff --git a/src/models/videochat_flash_processor.cpp b/src/models/videochat_flash_processor.cpp new file mode 100644 index 0000000000..65ac59d1f9 --- /dev/null +++ b/src/models/videochat_flash_processor.cpp @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "../generators.h" +#include "model.h" +#include "videochat_flash_processor.h" +#include + +namespace Generators { + +namespace { + +std::unique_ptr ConvertPixelValues(const OrtValue& float_tensor, + ONNXTensorElementDataType target_type, + Ort::Allocator& allocator) { + auto shape = float_tensor.GetTensorTypeAndShapeInfo()->GetShape(); + size_t count = float_tensor.GetTensorTypeAndShapeInfo()->GetElementCount(); + + if (target_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { + auto result = OrtValue::CreateTensor(allocator, shape); + std::copy(float_tensor.GetTensorData(), + float_tensor.GetTensorData() + count, + result->GetTensorMutableData()); + return result; + } + + std::unique_ptr result; + if (target_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16) { + result = OrtValue::CreateTensor(allocator, shape); + } else if (target_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) { + result = OrtValue::CreateTensor(allocator, shape); + } else { + throw std::runtime_error("Unsupported target type for pixel values conversion"); + } + + auto* cpu_device = GetDeviceInterface(DeviceType::CPU); + void* input_data = const_cast(static_cast(float_tensor.GetTensorData())); + void* output_data = result->GetTensorMutableRawData(); + cpu_device->Cast(input_data, output_data, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, target_type, count); + return result; +} + +// Build input_ids from prompt, inserting fixed_tokens_per_image <|image_pad|> tokens per image. +std::tuple, std::unique_ptr> +BuildPromptTokens(const Tokenizer& tokenizer, const std::string& prompt, + int64_t num_images, int64_t tokens_per_image, + Ort::Allocator& allocator) { + constexpr char vision_start_token[] = "<|vision_start|>"; + constexpr char vision_end_token[] = "<|vision_end|>"; + constexpr char image_pad_token[] = "<|image_pad|>"; + + std::string text = prompt; + int64_t total_image_tokens = num_images * tokens_per_image; + + // Verify prompt has the right number of vision_start markers + const std::regex vision_start_regex{R"(<\|vision_start\|>)"}; + auto begin = std::sregex_iterator(text.begin(), text.end(), vision_start_regex); + auto end = std::sregex_iterator(); + int64_t marker_count = std::distance(begin, end); + + if (num_images > 0 && marker_count != num_images) { + throw std::runtime_error("Prompt contained " + std::to_string(marker_count) + + " vision_start tokens but received " + std::to_string(num_images) + " images."); + } + + // Replace each <|vision_start|>...<|vision_end|> block with the correct pad count + if (num_images > 0) { + std::string modified; + size_t last_pos = 0; + std::string temp = text; + std::smatch match; + + while (std::regex_search(temp, match, vision_start_regex)) { + size_t abs_pos = match.position() + (text.size() - temp.size()); + modified += text.substr(last_pos, abs_pos - last_pos); + + modified += vision_start_token; + for (int64_t i = 0; i < tokens_per_image; ++i) + modified += image_pad_token; + modified += vision_end_token; + + last_pos = abs_pos + match.length(); + size_t ve_pos = text.find(vision_end_token, last_pos); + if (ve_pos != std::string::npos) + last_pos = ve_pos + strlen(vision_end_token); + + temp = match.suffix(); + } + modified += text.substr(last_pos); + text = modified; + } + + const std::vector input_ids = tokenizer.Encode(text.c_str()); + + auto input_ids_value = OrtValue::CreateTensor( + allocator, std::vector{1, static_cast(input_ids.size())}); + std::copy(input_ids.begin(), input_ids.end(), input_ids_value->GetTensorMutableData()); + + auto num_img_tokens = OrtValue::CreateTensor(allocator, std::vector{1}); + num_img_tokens->GetTensorMutableData()[0] = total_image_tokens; + + return {std::move(input_ids_value), std::move(num_img_tokens)}; +} + +} // namespace + +VideoChatFlashProcessor::VideoChatFlashProcessor(Config& config, const SessionInfo& session_info) + : pixel_values_type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT}, + num_visual_tokens_{config.model.vision.num_visual_tokens} { + if (num_visual_tokens_ <= 0) + throw std::runtime_error("videochat_flash_qwen requires vision.num_visual_tokens > 0 in genai_config.json"); + + const auto processor_config = (config.config_path / fs::path(config.model.vision.config_filename)).string(); + CheckResult(OrtxCreateProcessor(processor_.ToBeAssigned(), processor_config.c_str())); + + try { + pixel_values_type_ = session_info.GetInputDataType(config.model.vision.inputs.pixel_values); + } catch (...) { + } + + config.AddMapping(std::string(Config::Defaults::InputIdsName), config.model.embedding.inputs.input_ids); + config.AddMapping(std::string(Config::Defaults::PixelValuesName), config.model.vision.inputs.pixel_values); +} + +std::unique_ptr VideoChatFlashProcessor::Process(const Tokenizer& tokenizer, const Payload& payload) const { + std::string prompt = std::string(payload.prompt); + const Images* images = payload.images; + Ort::Allocator& allocator{Ort::Allocator::GetWithDefaultOptions()}; + auto named_tensors = std::make_unique(); + + // Text-only: no image processing needed + if (!images || images->num_images_ == 0) { + auto [input_ids, num_img_tokens] = BuildPromptTokens(tokenizer, prompt, 0, 0, allocator); + named_tensors->emplace(std::string(Config::Defaults::InputIdsName), + std::make_shared(std::move(input_ids))); + named_tensors->emplace(std::string(Config::Defaults::NumImageTokens), + std::make_shared(std::move(num_img_tokens))); + return named_tensors; + } + + // Run ORT Extensions image preprocessing (Decode → Resize → Rescale → Normalize) + ort_extensions::OrtxObjectPtr result; + CheckResult(OrtxImagePreProcess(processor_.get(), images->images_.get(), result.ToBeAssigned())); + + OrtxTensor* pixel_values = nullptr; + CheckResult(OrtxTensorResultGetAt(result.get(), 0, &pixel_values)); + + const float* pv_data{}; + const int64_t* pv_shape{}; + size_t pv_ndims; + CheckResult(OrtxGetTensorData(pixel_values, reinterpret_cast(&pv_data), + &pv_shape, &pv_ndims)); + + // Determine layout from ORT Extensions output (HWC format) + int64_t num_imgs, height, width, channels; + if (pv_ndims == 3) { + num_imgs = 1; + height = pv_shape[0]; + width = pv_shape[1]; + channels = pv_shape[2]; + } else if (pv_ndims == 4) { + num_imgs = pv_shape[0]; + height = pv_shape[1]; + width = pv_shape[2]; + channels = pv_shape[3]; + } else { + throw std::runtime_error("VideoChatFlashProcessor: unexpected pixel_values rank " + + std::to_string(pv_ndims) + " (expected 3 or 4)"); + } + + // Transpose HWC → CHW and reshape to [1, num_frames, C, H, W] + std::vector target_shape = {1, num_imgs, channels, height, width}; + auto float_tensor = OrtValue::CreateTensor(allocator, target_shape); + float* dst = float_tensor->GetTensorMutableData(); + + for (int64_t n = 0; n < num_imgs; ++n) { + const float* src_img = pv_data + n * height * width * channels; + float* dst_img = dst + n * channels * height * width; + for (int64_t c = 0; c < channels; ++c) { + for (int64_t h = 0; h < height; ++h) { + for (int64_t w = 0; w < width; ++w) { + dst_img[c * height * width + h * width + w] = src_img[h * width * channels + w * channels + c]; + } + } + } + } + + auto converted_pv = ConvertPixelValues(*float_tensor, pixel_values_type_, allocator); + named_tensors->emplace(std::string(Config::Defaults::PixelValuesName), + std::make_shared(std::move(converted_pv))); + + // Tokenize prompt with fixed visual token padding + auto [input_ids, num_img_tokens] = BuildPromptTokens( + tokenizer, prompt, static_cast(images->num_images_), + num_visual_tokens_, allocator); + named_tensors->emplace(std::string(Config::Defaults::InputIdsName), + std::make_shared(std::move(input_ids))); + named_tensors->emplace(std::string(Config::Defaults::NumImageTokens), + std::make_shared(std::move(num_img_tokens))); + + // Emit image_grid_thw for GetImageFeatureBatchSize to determine num_images. + // The pixel_values name is remapped (e.g. "pixel_values" → "images"), so the + // rank-based lookup in GetImageFeatureBatchSize won't match; it falls through + // to image_grid_thw whose name is not remapped. + auto grid_thw = OrtValue::CreateTensor(allocator, std::vector{num_imgs, 3}); + auto* grid_ptr = grid_thw->GetTensorMutableData(); + for (int64_t i = 0; i < num_imgs; ++i) { + grid_ptr[i * 3 + 0] = 1; + grid_ptr[i * 3 + 1] = height; + grid_ptr[i * 3 + 2] = width; + } + named_tensors->emplace("image_grid_thw", + std::make_shared(std::move(grid_thw))); + + return named_tensors; +} + +} // namespace Generators diff --git a/src/models/videochat_flash_processor.h b/src/models/videochat_flash_processor.h new file mode 100644 index 0000000000..b40576602d --- /dev/null +++ b/src/models/videochat_flash_processor.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "model.h" +#include "processor.h" +#include "ortx_processor.h" + +namespace Generators { + +struct VideoChatFlashProcessor : Processor { + VideoChatFlashProcessor(Config& config, const SessionInfo& session_info); + + std::unique_ptr Process(const Tokenizer& tokenizer, const Payload& payload) const override; + + private: + ort_extensions::OrtxObjectPtr processor_; + + ONNXTensorElementDataType pixel_values_type_; + int64_t num_visual_tokens_; +}; + +} // namespace Generators From 36ff41740456f1cca357c4b89b642bfb311ee639 Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Wed, 22 Apr 2026 06:29:54 -0500 Subject: [PATCH 10/22] Remove example folder --- examples/python/videochat-flash/README.md | 225 --------- .../python/videochat-flash/arch_readme.md | 435 ------------------ examples/python/videochat-flash/builder.py | 278 ----------- examples/python/videochat-flash/inference.py | 141 ------ .../videochat-flash/inference_ort_video.py | 353 -------------- .../videochat-flash/internVideo2_builder.py | 351 -------------- examples/python/videochat-flash/run.py | 119 ----- .../vcf-oga-fp32/genai_config.json | 70 --- .../vcf-oga-fp32/processor_config.json | 51 -- 9 files changed, 2023 deletions(-) delete mode 100644 examples/python/videochat-flash/README.md delete mode 100644 examples/python/videochat-flash/arch_readme.md delete mode 100644 examples/python/videochat-flash/builder.py delete mode 100644 examples/python/videochat-flash/inference.py delete mode 100644 examples/python/videochat-flash/inference_ort_video.py delete mode 100644 examples/python/videochat-flash/internVideo2_builder.py delete mode 100644 examples/python/videochat-flash/run.py delete mode 100644 examples/python/videochat-flash/vcf-oga-fp32/genai_config.json delete mode 100644 examples/python/videochat-flash/vcf-oga-fp32/processor_config.json diff --git a/examples/python/videochat-flash/README.md b/examples/python/videochat-flash/README.md deleted file mode 100644 index d2859097fc..0000000000 --- a/examples/python/videochat-flash/README.md +++ /dev/null @@ -1,225 +0,0 @@ -# VideoChat-Flash — OGA Export & Inference - -Model: [OpenGVLab/VideoChat-Flash-Qwen2\_5-7B\_InternVideo2-1B](https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B) - ---- - -## Architecture - -``` -Video frames / images - ↓ -InternVideo2-1B (39-block ViT, 3D spatiotemporal attention) - patch_embed → [cls_token + pos_embed + img_pos_embed] → blocks[0..38] - ↓ -mm_projector (2-layer MLP, mm_projector_type = tome16_mlp_hd64) - clip-level HiCo token compression → ~16 tokens / frame - ↓ -Embedding merger - model.embed_tokens(input_ids) with image-pad positions replaced by visual tokens - ↓ -Qwen2.5-7B decoder (28L, 28Q/4KV GQA, hidden=3584, rope_theta=1e6) - optional in-LLM HiCo video compression at layers 8, 16, 24 - (llm_compress_layer_list, llm_compress_type=attention, mm_llm_compress=False by default) - ↓ -logits -``` - -Key config values: -| Field | Value | -|---|---| -| `mm_vision_tower` | `internvideo2` (39 transformer blocks) | -| `mm_projector_type` | `tome16_mlp_hd64` (2-layer MLP) | -| `mm_hidden_size` | 1408 (vision → 3584 LM projection) | -| Image token ID | 151646 (`<\|image_pad\|>`) | -| `llm_compress_layer_list` | `[8, 16, 24]` (disabled by default) | - ---- - -## Phase 1 — Text Decoder (Done) - -### What was implemented - -| Component | File | Description | -|---|---|---| -| `VideoChatFlashQwenModel` | `src/python/py/models/builders/qwen.py` | Builder class inheriting `QwenModel`. Overrides `load_weights()` to load via `Qwen2ForCausalLM` (avoids video library imports from custom remote code) and `make_genai_config()` to bypass `AutoConfig` for the same reason. | -| Architecture mapping | `src/python/py/models/builder.py` | Maps `VideoChatFlashQwenForCausalLM` → `VideoChatFlashQwenModel`. Includes a config-bypass that peeks at `config.json` via `hf_hub_download` before invoking `AutoConfig`, preventing `av`/`cv2`/`decord` from being imported. | -| C++ model type | `src/models/model_type.h` | Registers `videochat_flash_qwen` in `IsVLM()`. Used when the full pipeline (vision + embedding + decoder) is present. | -| Export script | `examples/python/videochat-flash/builder.py` | `--text-only` exports the decoder with `input_ids` input and `type=qwen2` config (standalone mode). Full VLM export (`--no-text-only`) sets `type=videochat_flash_qwen` and `inputs_embeds` mode (Phase 2). | -| Inference script | `examples/python/videochat-flash/run.py` | Text-only inference using `generator.append_tokens()` and HF `AutoTokenizer` (OGA tokenizer backend is unsupported for this model). | - -### Design decisions - -- **`type=qwen2` in text-only mode**: `videochat_flash_qwen` triggers `MultiModalLanguageModel` in the OGA C++ runtime, which requires `vision.onnx` + `embedding.onnx`. Without them, model load fails. Text-only export uses `type=qwen2` (identical LM architecture) so it loads as a plain decoder. -- **`Qwen2ForCausalLM` weight loading**: The model's custom `modeling_videochat_flash.py` imports `av`, `cv2`, `decord`, `imageio`, and `timm` at module load time. These are video processing libraries not needed for the LM backbone. The builder bypasses them entirely by loading weights as `Qwen2ForCausalLM` directly. -- **Standard 2D RoPE**: Unlike Qwen2.5-VL which uses 3D MRoPE, VideoChat-Flash uses standard 2D RoPE (`rope_scaling=None`, `rope_theta=1e6`) — the decoder is reused unchanged from `QwenModel`. - -### Usage - -```bash -# Export text decoder (downloads from HuggingFace) -python builder.py --output ./vcf-oga-fp32 --text-only - -# Export from a local PyTorch checkpoint -python builder.py --input ./pytorch_vcf --output ./vcf-oga-fp32 --text-only - -# Run text inference -python run.py --model ./vcf-oga-fp32 --batch -python run.py --model ./vcf-oga-fp32 --prompt "What is the capital of France?" -``` - -### Validated - -- Exported: `model.onnx` (228 weights, 28-layer Qwen2.5-7B, ~14 GB fp32) -- Inference: correct answers on QA prompts using `conda`-installed OGA build (Python 3.11, `onnxruntime-genai 0.13.0.dev0`) - ---- - -## Phase 2 — Vision Encoder + Embedding Merger (TODO) - -### Overview - -The full VLM pipeline requires three ONNX models: - -``` -vision.onnx — InternVideo2-1B + mm_projector MLP -embedding.onnx — embed_tokens table + image-pad token replacement -model.onnx — Qwen2.5-7B decoder (inputs_embeds mode, already exported) -``` - -### 2.1 Vision Encoder (`vcf-vision.onnx`) - -**Source weights** (from `model.safetensors`): -- `model.vision_tower.vision_tower.*` — 39-block ViT (516 tensors) -- `model.mm_projector.mlp.*` — 2-layer MLP projector (4 tensors) - -**Export wrapper** (following `examples/python/qwen3-vl/builder.py` pattern): - -```python -class VisionExportWrapper(nn.Module): - def forward(self, pixel_values, num_frames): - # pixel_values: [T*N_patches, C*patch_size*patch_size] - # InternVideo2 encodes frames with 3D spatiotemporal attention - visual_tokens = self.vision_tower(pixel_values, num_frames) - # mm_projector: clip-level HiCo compression → ~16 tokens/frame - return self.mm_projector(visual_tokens) - -torch.onnx.export( - wrapper, - (pixel_values, num_frames), - "vcf-vision.onnx", - input_names=["pixel_values", "num_frames"], - output_names=["visual_tokens"], - dynamic_axes={ - "pixel_values": {0: "total_patches"}, - "visual_tokens": {0: "num_visual_tokens"}, - }, - opset_version=17, -) -``` - -**Key challenges:** -- InternVideo2 uses **3D spatiotemporal attention** (temporal + spatial patch tokens). Verify ONNX opset 17 supports the `einops.rearrange` and `timm` attention ops used. May require `einops` decomposition or custom ONNX ops. -- `img_pos_embed` (temporal position embedding) is dynamic based on `num_frames` — must be handled as a runtime input or computed inside the wrapper. -- The HiCo clip-level compression ratio (`tome16`) means the projector outputs ~16 tokens per frame regardless of input resolution. Verify this is deterministic (no graph-capture issues). -- `mm_projector_type = tome16_mlp_hd64`: ToMe (Token Merging) may involve non-trivial dynamic gather/scatter ops that need ONNX-compatible implementations. - -### 2.2 Embedding Merger (`vcf-embedding.onnx`) - -**Source weights**: `model.embed_tokens` (vocab embedding table, 152064 × 3584) - -**Export wrapper** (identical pattern to Qwen3-VL): - -```python -class EmbeddingWrapper(nn.Module): - IMAGE_TOKEN_ID = 151646 # <|image_pad|> - - def forward(self, input_ids, visual_tokens): - # input_ids: [1, seq_len] - # visual_tokens: [num_visual_tokens, 3584] (from vision.onnx) - inputs_embeds = self.embed_tokens(input_ids) # [1, seq_len, 3584] - vision_mask = (input_ids.view(-1) == self.IMAGE_TOKEN_ID) - inputs_embeds[0, vision_mask] = visual_tokens - return inputs_embeds # [1, seq_len, 3584] - -torch.onnx.export( - wrapper, - (input_ids, visual_tokens), - "vcf-embedding.onnx", - input_names=["input_ids", "visual_tokens"], - output_names=["inputs_embeds"], - dynamic_axes={ - "input_ids": {1: "seq_len"}, - "visual_tokens": {0: "num_visual_tokens"}, - }, - opset_version=17, -) -``` - -### 2.3 Re-export text decoder in VLM mode - -```bash -python builder.py --output ./vcf-oga-vlm --precision fp32 -# (omit --text-only → exclude_embeds=true, type=videochat_flash_qwen) -``` - -### 2.4 `genai_config.json` additions - -```json -"vision": { - "filename": "vcf-vision.onnx", - "inputs": { "pixel_values": "pixel_values", "num_frames": "num_frames" }, - "outputs": { "image_features": "visual_tokens" } -}, -"embedding": { - "filename": "vcf-embedding.onnx", - "inputs": { "input_ids": "input_ids", "image_features": "visual_tokens" }, - "outputs": { "inputs_embeds": "inputs_embeds" } -} -``` - -### 2.5 Video preprocessing pipeline - -Before the vision encoder, frames must be extracted and preprocessed. The model uses: -- `frame_aspect_ratio = square` (video frames are resized to squares) -- `image_aspect_ratio = anyres_nopad` (images use any-resolution tiling, no padding) -- `mm_spatial_pool_mode = bilinear` -- `mm_pos_num_frames = 8` (temporal position embeddings support up to 8 frames) - -A `vision_processor.json` (following `qwen3-vl` pattern) or a Python preprocessing script will be needed. - -### 2.6 In-LLM HiCo video compression (optional, advanced) - -When `mm_llm_compress = True`, layers `[8, 16, 24]` apply additional token compression inside the decoder using cross-attention with `mm_num_compress_latents = 128` learnable query tokens. This is disabled by default (`mm_llm_compress = False`) and can be ignored for Phase 2. - -If enabled in future, the compression layers would need to be exported as part of `model.onnx` or as separate side-car ONNX models. - ---- - -## File Map - -``` -examples/python/videochat-flash/ -├── builder.py # Export script (Phase 1: --text-only; Phase 2: full pipeline) -├── run.py # Text-only inference test -└── README.md # This file - -src/python/py/models/ -├── builder.py # create_model() dispatcher (VideoChatFlashQwenForCausalLM mapping) -└── builders/ - ├── qwen.py # VideoChatFlashQwenModel class - └── __init__.py # Export registration - -src/models/ -└── model_type.h # IsVLM(): videochat_flash_qwen registered -``` - ---- - -## Reference - -- Qwen3-VL export (closest working example): `examples/python/qwen3-vl/builder.py` -- OGA MultiModalLanguageModel runtime: `src/models/multi_modal.cpp` -- VLM model type dispatch: `src/models/model.cpp` (line ~1294) -- InternVideo2 paper: [https://arxiv.org/abs/2312.07514](https://arxiv.org/abs/2312.07514) -- VideoChat-Flash HF model: [https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B](https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B) diff --git a/examples/python/videochat-flash/arch_readme.md b/examples/python/videochat-flash/arch_readme.md deleted file mode 100644 index ef2ec120bb..0000000000 --- a/examples/python/videochat-flash/arch_readme.md +++ /dev/null @@ -1,435 +0,0 @@ -# Export Pipeline Architecture — `internVideo2_builder.py` - -Detailed architecture reference for the ONNX export of VideoChat-Flash's vision and embedding components. - -Source model: [`OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B`](https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B) - ---- - -## Table of Contents - -1. [Export Modes](#export-modes) -2. [Script Execution Flow](#script-execution-flow) -3. [Vision Export — `export_vision()`](#vision-export--export_vision) -4. [Embedding Export — `export_embedding()`](#embedding-export--export_embedding) -5. [Meta-Device Parameter Fix](#meta-device-parameter-fix) -6. [ONNX Weight Consolidation](#onnx-weight-consolidation) -7. [Full Shape Reference](#full-shape-reference) - ---- - -## Export Modes - -The script supports three mutually exclusive modes via CLI flags: - -``` -python internVideo2_builder.py # vision only (image mode) -python internVideo2_builder.py --video # vision only (video mode) -python internVideo2_builder.py --embed # vision + embedding -python internVideo2_builder.py --video --embed # vision (video) + embedding -python internVideo2_builder.py --embed-only # embedding only (no vision) -``` - -| Flag | Exports | Output files | -|------|---------|-------------| -| *(none)* | Vision (image) | `vcf-vision.onnx` + `.data` | -| `--video` | Vision (video) | `vcf-vision-video.onnx` + `.data` | -| `--embed` | Vision + Embedding | `vcf-vision*.onnx` + `vcf-embed.onnx` | -| `--embed-only` | Embedding only | `vcf-embed.onnx` + `.data` | - ---- - -## Script Execution Flow - -``` -┌──────────────────────────────────────────────────────────────────────────┐ -│ CLI Argument Parse │ -│ --video --embed --embed-only │ -└────────────────────────────┬─────────────────────────────────────────────┘ - │ - ┌──────────────┼──────────────┐ - ▼ ▼ ▼ - --embed-only --embed (default) - │ │ │ - │ ┌────┴────┐ │ - │ ▼ ▼ ▼ - │ export_vision() │ export_vision() - │ │ │ - │ ▼ │ - │ export_embedding() - │ │ - ▼ │ - export_embedding() │ - │ - ◄────┘ - Done -``` - ---- - -## Vision Export — `export_vision()` - -Exports the InternVideo2-1B vision tower and mm_projector (ToMe + MLP connector) as a single ONNX model. The mode (image vs. video) changes the wrapper class and compression behavior. - -### Step-by-step - -``` -[1/5] Load HF model (float16) - │ - ├── Extract vision_tower (InternVideo2-1B ViT) - ├── Extract mm_projector (ToMe token compression + MLP) - └── Delete LLM backbone (free ~7 GB: layers, embed_tokens, lm_head) - │ -[2/5] Fix meta-device parameters (see "Meta-Device Parameter Fix" below) - │ -[3/5] Wrap in mode-specific nn.Module, cast to float32, test forward pass - │ -[4/5] torch.onnx.export (opset 18, dynamic batch + frames) - │ -[5/5] Consolidate external weights (single .onnx.data file) -``` - -### Image Mode — `VisionWithProjectorImage` - -``` - images - [B, 1, 3, 224, 224] - │ - ▼ - ┌───────────────────────┐ - │ InternVideo2-1B │ - │ (vision_tower) │ - │ │ - │ ViT patch embed: │ - │ 224 / 14 = 16 │ - │ 16 × 16 = 256 │ - │ spatial patches │ - │ per frame │ - │ │ - │ hidden_dim = 1408 │ - └───────────┬───────────┘ - │ - [B, 256, 1408] - │ - ▼ - ┌───────────────────────┐ - │ mm_projector │ - │ │ - │ compress = False │ - │ ToMe: 4× merge │ - │ 256 → 64 tokens │ - │ │ - │ MLP: 1408 → 3584 │ - └───────────┬───────────┘ - │ - visual_tokens - [B, 64, 3584] -``` - -**Output:** `vcf-vision.onnx` — 64 visual tokens per image, matching `HIDDEN_SIZE=3584` of the Qwen2.5-7B LLM. - -### Video Mode — `VisionWithProjectorVideo` - -``` - images - [B, T=4, 3, 224, 224] - │ - ▼ - ┌───────────────────────┐ - │ InternVideo2-1B │ - │ (vision_tower) │ - │ │ - │ T frames × 256 │ - │ patches = 1024 │ - │ spatiotemporal │ - │ tokens │ - └───────────┬───────────┘ - │ - [B, T×256, 1408] - = [B, 1024, 1408] - │ - ▼ reshape - [B×T, 256, 1408] - = [4, 256, 1408] - │ - ▼ - ┌───────────────────────┐ - │ mm_projector │ - │ │ - │ compress = True │ - │ local_num_frames=T │ - │ ToMe: 16× merge │ - │ 256 → 16 tok/frame │ - │ │ - │ MLP: 1408 → 3584 │ - └───────────┬───────────┘ - │ - visual_tokens - [B, 16×T, 3584] - = [B, 64, 3584] -``` - -**Output:** `vcf-vision-video.onnx` — 64 visual tokens per 4-frame segment (16 tokens/frame, temporally compressed). - -**Important:** `T=4` (`mm_local_num_frames` from config) is baked as a constant in the reshape op at export time. At inference, each call must provide exactly T frames. Longer videos are processed as multiple segments. - -### ToMe Compression Comparison - -``` - Image Mode Video Mode - ┌──────────────┐ ┌──────────────┐ - Input patches │ 256 / frame │ │ 256 / frame │ - └──────┬───────┘ └──────┬───────┘ - │ │ - ▼ ▼ - ToMe merge 256 → 64 (4×) 256 → 16 (16×) per frame - ratio compress=False compress=True - │ │ - ▼ ▼ - Per call 64 tokens 16 × T = 64 tokens - (1 frame) (T=4 frames) - │ │ - ▼ ▼ - Tokens/frame 64 16 - Info density High spatial detail Temporal context, less spatial -``` - -### ONNX Dynamic Axes - -``` -input: "images" dim 0 = "batch" (variable) - dim 1 = "num_frames" (variable in graph, but T baked in reshape) - -output: "visual_tokens" dim 0 = "batch" (variable) - dim 1 = "num_visual_tokens" (variable) -``` - ---- - -## Embedding Export — `export_embedding()` - -Exports the `EmbeddingWithMerge` module: a Qwen2.5 embedding table that also injects visual tokens at `<|image_pad|>` positions. - -### Step-by-step - -``` -[1/3] Load embed_tokens.weight from safetensors (fp32, ~2 GB) - │ Only reads a single tensor — no full model load needed. - │ Scans shards for key "model.embed_tokens.weight" - │ -[2/3] Build EmbeddingWithMerge module, run test forward pass - │ Validates that <|image_pad|> positions are correctly replaced - │ -[3/3] torch.onnx.export → consolidate weights → vcf-embed.onnx -``` - -### `EmbeddingWithMerge` — Internal Data Flow - -``` - input_ids [B, seq_len] image_features [1, N, 3584] - │ │ - ▼ │ - ┌────────────────────────┐ │ - │ embed_tokens │ │ - │ nn.Embedding │ │ - │ (152064 × 3584) │ │ - └──────────┬─────────────┘ │ - │ │ - text_embeds │ - [B, seq_len, 3584] │ - │ │ - ▼ ▼ - ┌──────────────────────────────────────────────────────────┐ - │ Merge Logic │ - │ │ - │ 1. mask = (input_ids == 151655) bool [B, seq] │ - │ │ - │ 2. indices = cumsum(mask) - 1 int [B, seq] │ - │ Maps each <|image_pad|> to its │ - │ 0-based position in image_features │ - │ │ - │ 3. safe_features = cat( │ - │ image_features.flatten, [N, 3584] │ - │ zeros(1, 3584) ← safety row │ - │ ) [N+1, 3584] │ - │ │ - │ 4. visual_at_pos = embedding(indices, safe_features) │ - │ [B, seq, 3584] │ - │ │ - │ 5. output = where(mask, visual_at_pos, text_embeds) │ - │ │ - └──────────────────────────┬───────────────────────────────┘ - │ - inputs_embeds - [B, seq_len, 3584] -``` - -**Why the safety row?** When `image_features` has 0 tokens (text-only prompt), indices would index into an empty tensor. The appended zero row makes the gather always safe — those values are never selected by the `where` because `mask` is all-False. - -### ONNX Dynamic Axes - -``` -input: "input_ids" dim 0 = "batch" int64 - dim 1 = "seq_len" - -input: "image_features" dim 0 = "num_images" float32 - dim 1 = "num_image_tokens" - -output: "inputs_embeds" dim 0 = "batch" float32 - dim 1 = "seq_len" -``` - -All spatial dimensions are dynamic. `image_features` dim 1 can be 0 for text-only inference. - ---- - -## Meta-Device Parameter Fix - -The InternVideo2-1B checkpoint has a naming mismatch: - -``` - Checkpoint key: model.vision_tower.blocks.0.attn.proj.ls1.gamma - Model parameter: blocks.0.attn.proj.ls1.weight -``` - -`from_pretrained()` fails to match `ls1.gamma` → `ls1.weight`, leaving those `LayerScale` parameters on the `meta` device (empty placeholders). - -### Fix procedure (step [2/5]) - -``` -┌──────────────────────────────────────────────────────────────────┐ -│ 1. Identify meta-device params │ -│ {n: p for n, p in vision_tower.named_parameters() │ -│ if p.device.type == "meta"} │ -│ │ -│ 2. Download/locate safetensors shards │ -│ snapshot_download(model_id) │ -│ │ -│ 3. Build lookup: strip "model.vision_tower." prefix │ -│ from checkpoint keys │ -│ │ -│ 4. For each meta param, try matching: │ -│ param_name → direct match │ -│ param_name(.weight→.gamma) → gamma variant │ -│ │ -│ 5. Load matched tensor from safetensors shard │ -│ Set on parent module via setattr() │ -│ Cast to float16 to match model dtype │ -└──────────────────────────────────────────────────────────────────┘ -``` - ---- - -## ONNX Weight Consolidation - -`torch.onnx.export` with large models creates many individual external data files. The consolidation step (step [5/5]) merges them: - -``` -Before consolidation: After consolidation: - vcf-vision.onnx vcf-vision.onnx (graph only) - vision_tower.block0.weight vcf-vision.onnx.data (all weights) - vision_tower.block1.weight - mm_projector.mlp.0.weight - mm_projector.mlp.2.weight - ... (hundreds of files) -``` - -### Process - -``` -1. onnx.load(path, load_external_data=True) ← all weights into RAM -2. Delete individual weight files from disk -3. convert_model_to_external_data( - all_tensors_to_one_file=True, - location="*.onnx.data", - size_threshold=1024 ← only externalize tensors ≥1KB - ) -4. onnx.save_model() ← writes graph + single .data file -``` - ---- - -## Full Shape Reference - -### Constants - -``` -IMAGE_SIZE = 224 Input image resolution -PATCH_SIZE = 14 ViT patch size (224/14 = 16 grid) -PATCHES_PER_FRAME = 256 16 × 16 spatial patches -VIT_HIDDEN_SIZE = 1408 InternVideo2-1B hidden dimension -LLM_HIDDEN_SIZE = 3584 Qwen2.5-7B hidden dimension -LOCAL_NUM_FRAMES = 4 T — baked into video mode at export -VOCAB_SIZE = 152064 Qwen2.5 vocabulary size -IMAGE_PAD_TOKEN_ID = 151655 <|image_pad|> token id -``` - -### Image Mode — End-to-End Shapes - -``` - ┌────────────────────────────────────────┐ - │ vcf-vision.onnx │ -images ─────────────────▶│ │──────▶ visual_tokens -[B, 1, 3, 224, 224] │ ViT: [B, 256, 1408] │ [B, 64, 3584] - │ ToMe 4×: [B, 64, 1408] │ - │ MLP: [B, 64, 3584] │ - └────────────────────────────────────────┘ - - ┌────────────────────────────────────────┐ -input_ids ──────────────▶│ vcf-embed.onnx │ -[B, seq_len] │ │──────▶ inputs_embeds -image_features ─────────▶│ embed: [B, seq_len, 3584] │ [B, seq_len, 3584] -[1, 64, 3584] │ merge at <|image_pad|> positions │ - └────────────────────────────────────────┘ -``` - -### Video Mode — End-to-End Shapes - -``` - ┌────────────────────────────────────────┐ - │ vcf-vision-video.onnx │ -images ─────────────────▶│ │──────▶ visual_tokens -[B, 4, 3, 224, 224] │ ViT: [B, 1024, 1408] │ [B, 64, 3584] - │ reshape: [B×4, 256, 1408] │ - │ ToMe 16×: [B×4, 16, 1408] │ - │ MLP + rebatch: [B, 64, 3584] │ - └────────────────────────────────────────┘ - - ┌────────────────────────────────────────┐ -input_ids ──────────────▶│ vcf-embed.onnx │ -[B, seq_len] │ │──────▶ inputs_embeds -image_features ─────────▶│ embed: [B, seq_len, 3584] │ [B, seq_len, 3584] -[1, S×64, 3584] │ merge at <|image_pad|> positions │ - S = num_segments └────────────────────────────────────────┘ -``` - -### Multi-Segment Video (at inference time) - -``` - Video: 16 frames sampled - │ - ▼ group into segments of T=4 - Segment 0: frames [0,1,2,3] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] - Segment 1: frames [4,5,6,7] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] - Segment 2: frames [8,9,10,11] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] - Segment 3: frames [12,13,14,15] → [1, 4, 3, 224, 224] → vcf-vision-video → [1, 64, 3584] - │ - ▼ concatenate - visual_tokens [1, 256, 3584] (4 segments × 64 tokens) - │ - ▼ - Prompt needs 256 × <|image_pad|> tokens to match -``` - ---- - -## Output Files - -| File | Size | Contents | -|------|------|----------| -| `vcf-vision.onnx` | ~100 KB | ONNX graph (image mode) | -| `vcf-vision.onnx.data` | ~1 GB | InternVideo2-1B + mm_projector weights | -| `vcf-vision-video.onnx` | ~100 KB | ONNX graph (video mode) | -| `vcf-vision-video.onnx.data` | ~1 GB | Same weights, different graph topology | -| `vcf-embed.onnx` | ~100 KB | ONNX graph (embed + merge) | -| `vcf-embed.onnx.data` | ~2 GB | embed_tokens weight (152064 × 3584 × fp32) | diff --git a/examples/python/videochat-flash/builder.py b/examples/python/videochat-flash/builder.py deleted file mode 100644 index 995441616c..0000000000 --- a/examples/python/videochat-flash/builder.py +++ /dev/null @@ -1,278 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. -# Portions of this file consist of AI generated content. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -""" -Export VideoChat-Flash (OpenGVLab) ONNX models for onnxruntime-genai. - -Model: OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B -Architecture: - - Language backbone: Qwen2.5-7B (28L, GQA 28h/4kv, hidden=3584) - - Visual encoder: InternVideo2-1B (video ViT with 3D spatiotemporal attention) - - Connector: MLP-based HiCo token compression (~16 tokens/frame) - -This script exports: - 1. Text decoder (model.onnx) via OGA builder — fully functional - 2. Vision encoder (vcf-vision.onnx) — TODO: InternVideo2 export - 3. Embedding merger (vcf-embedding.onnx) — TODO: token fusion export - 4. genai_config.json — wired for OGA multimodal pipeline - -Phase 1 (this PR): Text decoder only. Vision/embedding stubs are included -as placeholders. Pass --text-only to skip vision/embedding export. - -Usage: - # Download model and export text decoder only (Phase 1): - python builder.py --output ./vcf-oga-fp32 --text-only - - # Full pipeline export (requires vision encoder work): - python builder.py --input ./pytorch_vcf --output ./vcf-oga-int4 - - # Text-only inference smoke test after export: - python builder.py --output ./vcf-oga-fp32 --text-only --run-e2e -""" - -import argparse -import json -import os -import sys - -# Use the local OGA model builder (src/python/py/models/builder.py) -# so our VideoChatFlashQwenModel registration is picked up. -_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) -sys.path.insert(0, os.path.join(_REPO_ROOT, "src", "python", "py", "models")) - -from builder import create_model # noqa: E402 (local OGA builder) - -HF_MODEL_ID = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" - -# VideoChat-Flash image placeholder token id (same vocab as Qwen2.5) -IMAGE_TOKEN_ID = 151646 # <|image_pad|> — verify against tokenizer_config.json - - -def prepare_model(input_dir): - """Load HF model config from local path or HuggingFace.""" - from transformers import AutoConfig - - print("\n[1/4] Loading model config...") - # trust_remote_code=False: config.json is standard JSON — no need for - # the custom modeling code (which requires av/cv2/decord). - config = AutoConfig.from_pretrained(input_dir, trust_remote_code=True) - print(f" architecture : {config.architectures[0]}") - print(f" hidden_size : {config.hidden_size}") - print(f" num_layers : {config.num_hidden_layers}") - print(f" num_heads : {config.num_attention_heads} Q / {config.num_key_value_heads} KV") - print(f" rope_theta : {config.rope_theta}") - print(f" vocab_size : {config.vocab_size}") - return config - - -def export_vision_model(model, config, output_dir): - """ - Export the InternVideo2-1B visual encoder. - - TODO (Phase 2): The InternVideo2 encoder uses 3D spatiotemporal attention - and learnable position embeddings. Export requires: - 1. Wrapping model.vision_tower (InternVideo2) in a torch.onnx-compatible - nn.Module that accepts (pixel_values: [T*H*W, C], grid_thw: [N, 3]). - 2. Exporting the HiCo clip-level MLP compression head. - 3. Verifying opset compatibility for temporal attention ops. - - For now this function is a no-op placeholder. - """ - print("\n[2/4] Vision encoder export — TODO (Phase 2, skipped)") - print(" The InternVideo2-1B encoder requires custom 3D spatiotemporal") - print(" attention export. See Phase 2 implementation plan.") - - -def export_embedding_model(model, config, output_dir): - """ - Export the embedding merger that fuses visual tokens into the token stream. - - TODO (Phase 2): VideoChat-Flash uses mm_patch_merge_type and llm_compress - layers to inject visual tokens. Export requires: - 1. Wrapping model.model.embed_tokens (Qwen2.5 embedding table). - 2. Implementing the token replacement mask (image_token_id → vision features). - 3. Handling the HiCo video-level compression in LLM layers (llm_compress_layer_list). - - For now this function is a no-op placeholder. - """ - print("\n[3/4] Embedding merger export — TODO (Phase 2, skipped)") - print(" Token fusion depends on Phase 2 vision encoder output shape.") - - -def export_text_model(input_dir, output_dir, precision, text_only): - """Export text decoder (Qwen2.5-7B backbone) via OGA builder.""" - print(f"\n[{3 if False else 2}/4] Exporting text decoder ({precision.upper()})...") - print(f" Source: {input_dir}") - - # create_model(model_name, input_path, ...): - # model_name — HF repo ID used for config/tokenizer lookup - # input_path — local dir (or HF repo ID when downloading) - # exclude_embeds controls decoder input: - # text_only → False (input_ids, standalone text inference) - # full VLM → True (inputs_embeds, from embedding merger) - hf_id = HF_MODEL_ID - local_or_hf = input_dir if input_dir is not None else hf_id - create_model( - hf_id, - local_or_hf, - output_dir, - precision, - "cpu", - os.path.join(output_dir, ".cache"), - exclude_embeds=not text_only, - ) - print(f" [OK] Text decoder: {os.path.join(output_dir, 'model.onnx')}") - - -def update_genai_config(output_dir, text_only): - """Patch genai_config.json with VideoChat-Flash model type and vision sections.""" - config_path = os.path.join(output_dir, "genai_config.json") - - with open(config_path, "r", encoding="utf-8") as f: - config = json.load(f) - - if text_only: - # Text-only / standalone mode: decoder takes input_ids, no vision/embedding. - # Use qwen2 type so OGA loads it as a plain decoder (MultiModalLanguageModel - # requires vision.onnx + embedding.onnx which are not present in this mode). - config["model"]["type"] = "qwen2" - else: - # Full VLM pipeline: decoder takes inputs_embeds from the embedding merger. - config["model"]["type"] = "videochat_flash_qwen" - - if not text_only: - # Phase 2: wire vision encoder and embedding merger - config["model"]["vision"] = { - "filename": "vcf-vision.onnx", - "inputs": { - "pixel_values": "pixel_values", - "image_grid_thw": "image_grid_thw", - }, - "outputs": { - "image_features": "visual_tokens", - }, - } - config["model"]["embedding"] = { - "filename": "vcf-embedding.onnx", - "inputs": { - "input_ids": "input_ids", - "image_features": "visual_tokens", - }, - "outputs": { - "inputs_embeds": "inputs_embeds", - }, - } - - with open(config_path, "w", encoding="utf-8") as f: - json.dump(config, f, indent=2) - - print(f" [OK] Updated: genai_config.json (type={config['model']['type']})") - - -def run_e2e_smoke(output_dir, prompt): - """Quick text-only inference smoke test.""" - import onnxruntime_genai as og - - print("\n[Smoke] Running text-only inference...") - model = og.Model(output_dir) - tokenizer = og.Tokenizer(model) - tokens = tokenizer.encode(prompt) - - params = og.GeneratorParams(model) - params.set_search_options(max_length=128) - params.input_ids = tokens - - generator = og.Generator(model, params) - print("Output:", end=" ", flush=True) - while not generator.is_done(): - generator.generate_next_token() - token = generator.get_next_tokens()[0] - print(tokenizer.decode([token]), end="", flush=True) - print() - - -def main(): - parser = argparse.ArgumentParser( - description="Export VideoChat-Flash for onnxruntime-genai" - ) - parser.add_argument( - "--input", - type=str, - default=None, - help="Local PyTorch model directory. If omitted, downloads from HuggingFace.", - ) - parser.add_argument( - "--output", - type=str, - default="./vcf-oga", - help="Output directory for ONNX models", - ) - parser.add_argument( - "--precision", - type=str, - default="fp32", - choices=["fp32", "fp16", "int4"], - help="Text decoder precision", - ) - parser.add_argument( - "--text-only", - action="store_true", - help="Export text decoder only (Phase 1). Skip vision/embedding export.", - ) - parser.add_argument( - "--run-e2e", - action="store_true", - help="Run text-only inference smoke test after export.", - ) - parser.add_argument( - "--prompt", - type=str, - default="Describe the video in one sentence.", - help="Prompt for --run-e2e smoke test", - ) - args = parser.parse_args() - - input_dir = args.input or HF_MODEL_ID - output_dir = os.path.abspath(args.output) - os.makedirs(output_dir, exist_ok=True) - - print("=" * 70) - print("VideoChat-Flash ONNX Export for OGA") - print("=" * 70) - print(f" Source : {input_dir}") - print(f" Output : {output_dir}") - print(f" Precision: {args.precision.upper()}") - print(f" Mode : {'text-only (Phase 1)' if args.text_only else 'full pipeline (Phase 2)'}") - - if not args.text_only: - # Phase 2: load full model weights for vision/embedding export - prepare_model(input_dir) - export_vision_model(None, None, output_dir) - export_embedding_model(None, None, output_dir) - - export_text_model(input_dir, output_dir, args.precision, args.text_only) - - print("\n[4/4] Updating genai_config.json...") - update_genai_config(output_dir, args.text_only) - - if args.run_e2e: - run_e2e_smoke(output_dir, args.prompt) - - print("\n" + "=" * 70) - print("[SUCCESS] Export complete!") - print("=" * 70) - print(f"\nOutput: {output_dir}") - print("\nExported files:") - print(f" model.onnx ({args.precision.upper()}, text decoder — Qwen2.5-7B)") - if not args.text_only: - print(" vcf-vision.onnx (Phase 2 TODO — InternVideo2-1B)") - print(" vcf-embedding.onnx (Phase 2 TODO — token merger)") - print(" genai_config.json (type=videochat_flash_qwen)") - print() - - -if __name__ == "__main__": - main() diff --git a/examples/python/videochat-flash/inference.py b/examples/python/videochat-flash/inference.py deleted file mode 100644 index c4bb0f48d1..0000000000 --- a/examples/python/videochat-flash/inference.py +++ /dev/null @@ -1,141 +0,0 @@ -import argparse -import json -import onnxruntime_genai as og - -def main(): - parser = argparse.ArgumentParser( - description="ONNX Runtime GenAI inference for Qwen3-VL" - ) - - parser.add_argument( - "--model_path", - type=str, - default="cpu_and_mobile/models", - help="Path to the model directory containing genai_config.json and ONNX models" - ) - parser.add_argument( - "--image", - type=str, - default=None, - help="Path to image file" - ) - parser.add_argument( - "--prompt", - type=str, - default=None, - help="Text prompt" - ) - parser.add_argument( - "--interactive", - action="store_true", - help="Run in interactive mode" - ) - - args = parser.parse_args() - - # Load model - print(f"Loading model from: {args.model_path}") - model = og.Model(args.model_path) - processor = model.create_multimodal_processor() - tokenizer = og.Tokenizer(model) - tokenizer_stream = processor.create_stream() - - if args.interactive: - interactive_mode(model, processor, tokenizer, tokenizer_stream, args) - elif args.prompt: - generate_response(model, processor, tokenizer, tokenizer_stream, args.prompt, args.image) - else: - print("Please provide --prompt or use --interactive mode") - parser.print_help() - - -def generate_response(model, processor, tokenizer, tokenizer_stream, prompt, image_path): - # Build messages for chat template - images = None - if image_path: - print(f"Loading image: {image_path}") - images = og.Images.open(image_path) - # The embedding model replaces <|image_pad|> positions with visual features. - # We need exactly 64 pad tokens (one per visual token from the vision model). - NUM_VISUAL_TOKENS = 64 - image_pads = "<|image_pad|>" * NUM_VISUAL_TOKENS - messages = [ - { - "role": "user", - "content": f"<|vision_start|>{image_pads}<|vision_end|>\n{prompt}" - } - ] - else: - messages = [ - { - "role": "user", - "content": prompt - } - ] - - full_prompt = tokenizer.apply_chat_template(json.dumps(messages), add_generation_prompt=True) - - print(f"\nPrompt: {prompt}") - print("Generating response...") - - inputs = processor(full_prompt, images=images) - - params = og.GeneratorParams(model) - params.set_search_options(max_length=4096) - - generator = og.Generator(model, params) - generator.set_inputs(inputs) - - print("\nResponse: ", end="", flush=True) - while not generator.is_done(): - generator.generate_next_token() - new_token = generator.get_next_tokens()[0] - print(tokenizer_stream.decode(new_token), end="", flush=True) - print() - del generator - - -def interactive_mode(model, processor, tokenizer, tokenizer_stream, args): - """Run in interactive mode.""" - print("\n" + "="*50) - print("Interactive Mode - Enter 'quit' or 'exit' to stop") - print("To include an image, type: image:/path/to/image.jpg") - print("="*50 + "\n") - - while True: - try: - user_input = input("You: ").strip() - except EOFError: - break - - if user_input.lower() in ['quit', 'exit']: - break - if not user_input: - print("Please enter a prompt.") - continue - - # Check for image path - image_path = None - prompt = user_input - if user_input.startswith("image:"): - parts = user_input.split(" ", 1) - image_path = parts[0][6:] # Remove "image:" prefix - prompt = parts[1] if len(parts) > 1 else "Describe this image" - - try: - generate_response( - model, processor, tokenizer, tokenizer_stream, - prompt, image_path - ) - except Exception as e: - print(f"Error: {e}") - import traceback - traceback.print_exc() - - print("-"*50 + "\n") - - print("Goodbye!") - - -if __name__ == "__main__": - main() diff --git a/examples/python/videochat-flash/inference_ort_video.py b/examples/python/videochat-flash/inference_ort_video.py deleted file mode 100644 index 3819dcc623..0000000000 --- a/examples/python/videochat-flash/inference_ort_video.py +++ /dev/null @@ -1,353 +0,0 @@ -""" -VideoChat-Flash **video** inference using pure ONNX Runtime. -Handles video frame extraction, vision encoding (with temporal compression), -embedding merge, and decoder KV-cache autoregressive decoding directly. - -Memory strategy: only one large model is loaded at a time. - Phase 1 – Vision (~1GB): load → run all segments → free - Phase 2 – Embed (~2GB): load → run initial prompt → extract weight → free - Phase 3 – Decoder (~14GB): load → autoregressive decode - -Models needed in --model_path: - vcf-vision-video.onnx + vcf-vision-video.onnx.data (InternVideo2 + mm_projector, compress=True) - vcf-embed.onnx + vcf-embed.onnx.data (embedding + visual merge) - model.onnx + model.onnx.data (Qwen2.5-7B decoder) - -The video vision model was exported with local_num_frames=4 (T=4). -Each segment of T frames produces 16*T = 64 visual tokens via ToMe compression. -For a video sampled at N total frames → ceil(N/T) segments → ceil(N/T)*64 visual tokens. - -Usage: - python inference_ort_video.py --model_path ./vcf-oga-fp32 --video video.mp4 --prompt "Describe this video" - python inference_ort_video.py --model_path ./vcf-oga-fp32 --video video.mp4 --num_frames 16 --prompt "What happens?" - python inference_ort_video.py --model_path ./vcf-oga-fp32 --image cat.jpeg --prompt "Describe this image" - python inference_ort_video.py --model_path ./vcf-oga-fp32 --prompt "Hello, who are you?" -""" - -import argparse -import gc -import math -import os -import numpy as np -import onnxruntime as ort -from transformers import AutoTokenizer -from PIL import Image - -MODEL_ID = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" -IMAGE_PAD_ID = 151655 # <|image_pad|> -VISION_START_ID = 151652 # <|vision_start|> -VISION_END_ID = 151653 # <|vision_end|> -EOS_TOKEN_ID = 151645 # <|im_end|> -LOCAL_NUM_FRAMES = 4 # T — baked into vcf-vision-video.onnx at export time -TOKENS_PER_SEGMENT = 64 # 16 * T with compress=True -NUM_LAYERS = 28 -NUM_KV_HEADS = 4 -HEAD_SIZE = 128 -HIDDEN_SIZE = 3584 - -IMAGE_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) -IMAGE_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) -IMAGE_SIZE = 224 - - -def preprocess_frame(frame_rgb): - """Resize and normalize a single RGB frame (PIL Image or ndarray) → [3, 224, 224].""" - if isinstance(frame_rgb, np.ndarray): - frame_rgb = Image.fromarray(frame_rgb) - frame_rgb = frame_rgb.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BICUBIC) - pixels = np.array(frame_rgb, dtype=np.float32) / 255.0 - pixels = (pixels - IMAGE_MEAN) / IMAGE_STD - return pixels.transpose(2, 0, 1) # HWC → CHW - - -def extract_video_frames(video_path, num_frames): - """Extract `num_frames` uniformly-spaced RGB frames from a video file. - - Returns a list of PIL Images. - """ - import cv2 - - cap = cv2.VideoCapture(video_path) - if not cap.isOpened(): - raise RuntimeError(f"Cannot open video: {video_path}") - - total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - fps = cap.get(cv2.CAP_PROP_FPS) - if total <= 0: - raise RuntimeError(f"Video has 0 frames: {video_path}") - - breakpoint() - - sample_count = min(num_frames, total) - indices = np.linspace(0, total - 1, sample_count, dtype=int) - - frames = [] - for idx in indices: - cap.set(cv2.CAP_PROP_POS_FRAMES, int(idx)) - ret, bgr = cap.read() - if not ret: - continue - rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB) - frames.append(Image.fromarray(rgb)) - - cap.release() - print(f" Video: {total} total frames, {fps:.1f} fps, sampled {len(frames)} frames") - return frames - - -def prepare_video_segments(frames): - """Group preprocessed frames into segments of LOCAL_NUM_FRAMES (T=4). - - If the frame count isn't divisible by T, the last segment is padded by - repeating the final frame. - - Returns: np.ndarray [num_segments, T, 3, 224, 224] - """ - breakpoint() - preprocessed = np.stack([preprocess_frame(f) for f in frames]) # [N, 3, H, W] - N = preprocessed.shape[0] - - T = LOCAL_NUM_FRAMES - num_segments = math.ceil(N / T) - padded_len = num_segments * T - - if padded_len > N: - pad = np.stack([preprocessed[-1]] * (padded_len - N)) - preprocessed = np.concatenate([preprocessed, pad], axis=0) - - segments = preprocessed.reshape(num_segments, T, 3, IMAGE_SIZE, IMAGE_SIZE) - return segments - - -def preprocess_image(image_path): - """Preprocess a single image → [1, T, 3, 224, 224] (repeat to fill one segment).""" - img = Image.open(image_path).convert("RGB") - frame = preprocess_frame(img) - segment = np.stack([frame] * LOCAL_NUM_FRAMES) # [T, 3, H, W] - return segment[np.newaxis, :, :, :, :] # [1, T, 3, H, W] - - -def build_prompt(tokenizer, user_prompt, num_visual_tokens): - """Build tokenized prompt with chat template and the correct number of pad tokens.""" - if num_visual_tokens > 0: - image_pads = "<|image_pad|>" * num_visual_tokens - content = f"<|vision_start|>{image_pads}<|vision_end|>\n{user_prompt}" - else: - content = user_prompt - - messages = [{"role": "user", "content": content}] - text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) - input_ids = tokenizer.encode(text, return_tensors="np").astype(np.int64) - return input_ids # [1, seq_len] - - -def run_vision_segments(session, segments): - """Run vision ONNX on each segment and concatenate visual tokens. - - segments: [num_segments, T, 3, H, W] - Returns: [1, num_segments * TOKENS_PER_SEGMENT, HIDDEN_SIZE] - """ - all_tokens = [] - for i in range(segments.shape[0]): - seg = segments[i:i+1] # [1, T, 3, H, W] - outputs = session.run(None, {"images": seg}) - all_tokens.append(outputs[0]) # [1, 64, 3584] - - visual_tokens = np.concatenate(all_tokens, axis=1) # [1, total_tokens, 3584] - return visual_tokens - - -def run_embedding(session, input_ids, image_features): - """Run embedding ONNX: input_ids + image_features → inputs_embeds.""" - outputs = session.run(None, { - "input_ids": input_ids, - "image_features": image_features, - }) - return outputs[0] # [1, seq_len, 3584] - - -def extract_embed_weight(model_dir): - """Extract embed_tokens.weight from the ONNX external data file. - - Loads only the lightweight graph protobuf, reads the offset/length of the - embedding initializer, then reads just that slice from the .onnx.data file. - """ - import onnx - - graph_path = os.path.join(model_dir, "vcf-embed.onnx") - model_proto = onnx.load(graph_path, load_external_data=False) - - for init in model_proto.graph.initializer: - if "embed_tokens.weight" in init.name: - ext = {e.key: e.value for e in init.external_data} - location = ext["location"] - offset = int(ext.get("offset", 0)) - length = int(ext["length"]) - shape = tuple(init.dims) - - data_path = os.path.join(model_dir, location) - with open(data_path, "rb") as f: - f.seek(offset) - raw = f.read(length) - - del model_proto - return np.frombuffer(raw, dtype=np.float32).reshape(shape).copy() - - raise ValueError("embed_tokens.weight not found in vcf-embed.onnx") - - -def embed_token_np(embed_weight, token_id): - """Numpy-based single-token embedding lookup → [1, 1, hidden_size].""" - return embed_weight[token_id][np.newaxis, np.newaxis, :] - - -def greedy_decode(decoder_session, embed_weight, inputs_embeds, max_new_tokens=512): - """Autoregressive decoding with KV cache.""" - batch = 1 - seq_len = inputs_embeds.shape[1] - - past_kv = {} - for i in range(NUM_LAYERS): - past_kv[f"past_key_values.{i}.key"] = np.zeros( - (batch, NUM_KV_HEADS, 0, HEAD_SIZE), dtype=np.float32 - ) - past_kv[f"past_key_values.{i}.value"] = np.zeros( - (batch, NUM_KV_HEADS, 0, HEAD_SIZE), dtype=np.float32 - ) - - attention_mask = np.ones((batch, seq_len), dtype=np.int64) - current_embeds = inputs_embeds - - for step in range(max_new_tokens): - feeds = {"inputs_embeds": current_embeds, "attention_mask": attention_mask} - feeds.update(past_kv) - - outputs = decoder_session.run(None, feeds) - - logits = outputs[0] - next_token = int(np.argmax(logits[0, -1, :])) - - if next_token == EOS_TOKEN_ID: - break - - for i in range(NUM_LAYERS): - past_kv[f"past_key_values.{i}.key"] = outputs[1 + i] - past_kv[f"past_key_values.{i}.value"] = outputs[1 + NUM_LAYERS + i] - - total_len = past_kv["past_key_values.0.key"].shape[2] - attention_mask = np.ones((batch, total_len + 1), dtype=np.int64) - - current_embeds = embed_token_np(embed_weight, next_token) - - yield next_token - - -def main(): - parser = argparse.ArgumentParser( - description="VideoChat-Flash video inference (pure ONNX Runtime)" - ) - parser.add_argument("--model_path", type=str, required=True) - parser.add_argument("--video", type=str, default=None, help="Path to video file") - parser.add_argument("--image", type=str, default=None, help="Path to image file (single-frame mode)") - parser.add_argument("--num_frames", type=int, default=28, - help="Number of frames to sample from video (rounded up to multiple of T=4)") - parser.add_argument("--prompt", type=str, required=True) - parser.add_argument("--max_tokens", type=int, default=1024) - args = parser.parse_args() - - if args.video and args.image: - print("ERROR: specify either --video or --image, not both") - return - - print(f"onnxruntime version: {ort.__version__}") - so = ort.SessionOptions() - so.log_severity_level = 3 - so.enable_cpu_mem_arena = False - - model_dir = args.model_path - has_visual = args.video is not None or args.image is not None - - print("Loading tokenizer...") - tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True) - - # ---- Phase 1: Vision (load → run → free) ---- - visual_tokens = np.zeros((1, 0, HIDDEN_SIZE), dtype=np.float32) - num_visual_tokens = 0 - - if has_visual: - vision_path = os.path.join(model_dir, "vcf-vision-video.onnx") - if not os.path.exists(vision_path): - print(f"ERROR: vcf-vision-video.onnx not found in {model_dir}") - return - - print("\nLoading video vision model...") - vision_session = ort.InferenceSession(vision_path, so) - print(" Loaded: vcf-vision-video.onnx") - - if args.video: - print(f"Extracting frames from: {args.video}") - frames = extract_video_frames(args.video, args.num_frames) - if len(frames) == 0: - print("ERROR: no frames extracted from video") - return - - print(f"Preparing segments (T={LOCAL_NUM_FRAMES})...") - segments = prepare_video_segments(frames) - num_segments = segments.shape[0] - print(f" {len(frames)} frames → {num_segments} segment(s) of {LOCAL_NUM_FRAMES} frames") - del frames - else: - print(f"Preprocessing image (single-frame mode): {args.image}") - segments = preprocess_image(args.image) - num_segments = 1 - print(f" 1 image → 1 segment ({LOCAL_NUM_FRAMES} repeated frames)") - - print("Running video vision model...") - visual_tokens = run_vision_segments(vision_session, segments) - num_visual_tokens = visual_tokens.shape[1] - print(f" Visual tokens: {visual_tokens.shape} ({num_segments} seg × {TOKENS_PER_SEGMENT} tok)") - - del vision_session, segments - gc.collect() - print(" Vision session freed.") - - # ---- Phase 2: Embedding (load → run prompt → free session) ---- - print("\nLoading embedding model...") - embed_session = ort.InferenceSession( - os.path.join(model_dir, "vcf-embed.onnx"), so - ) - print(" Loaded: vcf-embed.onnx") - - print(f"\nPrompt: {args.prompt}") - input_ids = build_prompt(tokenizer, args.prompt, num_visual_tokens) - print(f" Token count: {input_ids.shape[1]} (includes {num_visual_tokens} image_pad tokens)") - - print("Running embedding (merges visual tokens into prompt)...") - inputs_embeds = run_embedding(embed_session, input_ids, visual_tokens) - print(f" inputs_embeds: {inputs_embeds.shape}") - - del embed_session, visual_tokens, input_ids - gc.collect() - print(" Embedding session freed.") - - print("Extracting embedding weight table from ONNX data file...") - embed_weight = extract_embed_weight(model_dir) - print(f" embed_weight: {embed_weight.shape} ({embed_weight.nbytes / 1e9:.2f} GB)") - - # ---- Phase 3: Decoder (load last, largest model) ---- - print("\nLoading decoder model...") - decoder_session = ort.InferenceSession( - os.path.join(model_dir, "model.onnx"), so - ) - print(" Loaded: model.onnx") - - print("\nResponse: ", end="", flush=True) - for token_id in greedy_decode(decoder_session, embed_weight, inputs_embeds, args.max_tokens): - text = tokenizer.decode([token_id], skip_special_tokens=False) - print(text, end="", flush=True) - print() - print("\nDone.") - - -if __name__ == "__main__": - main() diff --git a/examples/python/videochat-flash/internVideo2_builder.py b/examples/python/videochat-flash/internVideo2_builder.py deleted file mode 100644 index f8b3b3b73e..0000000000 --- a/examples/python/videochat-flash/internVideo2_builder.py +++ /dev/null @@ -1,351 +0,0 @@ -import torch -import torch.nn as nn -import gc -import os -import argparse - -model_id = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" - -parser = argparse.ArgumentParser() -parser.add_argument("--video", action="store_true", help="Export in video mode (compress=True, T=local_num_frames)") -parser.add_argument("--embed", action="store_true", help="Also export the text embedding layer (embed_tokens)") -parser.add_argument("--embed-only", action="store_true", help="Export ONLY the embedding layer (skip vision export)") -args = parser.parse_args() - - -IMAGE_PAD_TOKEN_ID = 151655 # <|image_pad|> - - -class EmbeddingWithMerge(nn.Module): - """Embedding lookup + visual feature injection at <|image_pad|> positions. - - Inputs: input_ids [batch, seq_len], image_features [1, num_visual_tokens, hidden_size] - Output: inputs_embeds [batch, seq_len, hidden_size] - - At positions where input_ids == image_pad_id, the text embedding is - replaced by the corresponding visual feature from image_features. - For text-only prompts (no image_pad tokens), image_features can be - empty [1, 0, hidden_size] and the output is pure text embeddings. - """ - - def __init__(self, embed_weight, image_pad_id=IMAGE_PAD_TOKEN_ID): - super().__init__() - vocab_size, embed_dim = embed_weight.shape - self.embed_tokens = nn.Embedding(vocab_size, embed_dim) - self.embed_tokens.weight = nn.Parameter(embed_weight) - self.image_pad_id = image_pad_id - - def forward(self, input_ids, image_features): - text_embeds = self.embed_tokens(input_ids) - hidden_size = text_embeds.shape[-1] - - mask = (input_ids == self.image_pad_id) - - # Map each image_pad position to its 0-based index in image_features - indices = mask.long().cumsum(dim=-1) - 1 - indices = indices.clamp(min=0) - - # Flatten visual features; append a dummy zero row so indexing is - # always safe (text-only case: image_features has 0 tokens) - flat_features = image_features.reshape(-1, hidden_size) - safe_features = torch.cat([ - flat_features, - torch.zeros(1, hidden_size, dtype=flat_features.dtype, device=flat_features.device) - ], dim=0) - - visual_at_positions = torch.nn.functional.embedding(indices, safe_features) - - mask_3d = mask.unsqueeze(-1).expand_as(text_embeds) - inputs_embeds = torch.where(mask_3d, visual_at_positions, text_embeds) - return inputs_embeds - - -def export_embedding(): - """Export embedding+merge model as fp32 ONNX. - Loads only the embedding weight from safetensors — no full model needed (~2GB RAM). - """ - import onnx - from onnx.external_data_helper import convert_model_to_external_data - from safetensors import safe_open - from huggingface_hub import snapshot_download - from transformers import AutoConfig - - print("[1/3] Loading embedding weight from safetensors (fp32)...") - config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) - vocab_size = config.vocab_size - embed_dim = config.hidden_size - - model_dir = snapshot_download(model_id) - - embed_key = "model.embed_tokens.weight" - embed_weight = None - for fname in sorted(os.listdir(model_dir)): - if not fname.endswith(".safetensors"): - continue - shard_path = os.path.join(model_dir, fname) - with safe_open(shard_path, framework="pt", device="cpu") as f: - if embed_key in f.keys(): - embed_weight = f.get_tensor(embed_key).float() - print(f" Loaded {embed_key} from {fname}: {embed_weight.shape} → fp32") - break - - if embed_weight is None: - print(f" ERROR: Could not find {embed_key} in safetensors shards") - return - - model = EmbeddingWithMerge(embed_weight, image_pad_id=IMAGE_PAD_TOKEN_ID) - model.eval() - del embed_weight - print(f" embed_tokens: vocab_size={vocab_size:,}, dim={embed_dim}") - print(f" image_pad_id: {IMAGE_PAD_TOKEN_ID} (<|image_pad|>)") - - # Test: simulate a prompt with 64 image_pad tokens - NUM_VISUAL_TOKENS = 64 - print(f"\n[2/3] Running test forward pass...") - dummy_ids = torch.ones(1, 10 + NUM_VISUAL_TOKENS, dtype=torch.long) * 100 - dummy_ids[0, 5:5 + NUM_VISUAL_TOKENS] = IMAGE_PAD_TOKEN_ID - dummy_features = torch.randn(1, NUM_VISUAL_TOKENS, embed_dim) - - with torch.no_grad(): - test_out = model(dummy_ids, dummy_features) - print(f" input_ids: {dummy_ids.shape} (with {NUM_VISUAL_TOKENS} image_pad tokens)") - print(f" image_features: {dummy_features.shape}") - print(f" inputs_embeds: {test_out.shape} (dtype={test_out.dtype})") - - # Verify merge: image_pad positions should have visual features, not text embeds - text_only = model.embed_tokens(dummy_ids) - merged_at_pad = test_out[0, 5] - text_at_pad = text_only[0, 5] - visual_expected = dummy_features[0, 0] - assert torch.allclose(merged_at_pad, visual_expected), "Merge verification failed!" - assert not torch.allclose(merged_at_pad, text_at_pad), "Merge did not replace text embed!" - print(" Merge verification: PASSED") - - print(f"\n[3/3] Exporting to ONNX...") - embed_onnx = "vcf-embed.onnx" - embed_data = "vcf-embed.onnx.data" - - with torch.no_grad(): - torch.onnx.export( - model, - (dummy_ids, dummy_features), - embed_onnx, - input_names=["input_ids", "image_features"], - output_names=["inputs_embeds"], - dynamic_axes={ - "input_ids": {0: "batch", 1: "seq_len"}, - "image_features": {0: "num_images", 1: "num_image_tokens"}, - "inputs_embeds": {0: "batch", 1: "seq_len"}, - }, - opset_version=18, - dynamo=False, - ) - - del model - gc.collect() - - embed_proto = onnx.load(embed_onnx, load_external_data=True) - - for f_name in os.listdir("."): - if f_name.endswith((".onnx", ".onnx.data", ".py", ".json")): - continue - if os.path.isfile(f_name) and not f_name.startswith("."): - _, ext = os.path.splitext(f_name) - if ext == "": - os.remove(f_name) - - convert_model_to_external_data( - embed_proto, - all_tensors_to_one_file=True, - location=embed_data, - size_threshold=1024, - convert_attribute=False, - ) - onnx.save_model(embed_proto, embed_onnx) - print(f"\nExported {embed_onnx} + {embed_data} successfully") - print(f" Inputs: input_ids [B, seq_len] + image_features [1, N, {embed_dim}]") - print(f" Output: inputs_embeds [B, seq_len, {embed_dim}] (fp32)") - print(f" Merges visual tokens at <|image_pad|> (id={IMAGE_PAD_TOKEN_ID}) positions") - - -def export_vision(): - """Export vision tower + mm_projector as an ONNX model.""" - import onnx - from onnx.external_data_helper import convert_model_to_external_data - from transformers import AutoModel, AutoConfig - - config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) - LOCAL_NUM_FRAMES = getattr(config, "mm_local_num_frames", 4) - - class VisionWithProjectorImage(nn.Module): - def __init__(self, vision_tower, mm_projector): - super().__init__() - self.vision_tower = vision_tower - self.mm_projector = mm_projector - - def forward(self, images): - visual_features = self.vision_tower(images) - projected = self.mm_projector(visual_features, compress=False) - return projected - - class VisionWithProjectorVideo(nn.Module): - def __init__(self, vision_tower, mm_projector, local_num_frames): - super().__init__() - self.vision_tower = vision_tower - self.mm_projector = mm_projector - self.local_num_frames = local_num_frames - - def forward(self, images): - T = self.local_num_frames - visual_features = self.vision_tower(images) - B = visual_features.shape[0] - visual_features = visual_features.reshape(B * T, -1, visual_features.shape[-1]) - projected = self.mm_projector(visual_features, compress=True, local_num_frames=T) - return projected - - print("[1/5] Loading model in float16...") - model = AutoModel.from_pretrained( - model_id, - trust_remote_code=True, - torch_dtype=torch.float16, - low_cpu_mem_usage=False, - ) - - vision_tower = model.get_vision_tower() - mm_projector = model.model.mm_projector - - del model.model.layers, model.model.embed_tokens, model.lm_head - del model - gc.collect() - - meta_params = {n: p for n, p in vision_tower.named_parameters() if p.device.type == "meta"} - if meta_params: - print(f"[2/5] Fixing {len(meta_params)} meta-device params (gamma→weight name mismatch)...") - - from safetensors import safe_open - from huggingface_hub import snapshot_download - - model_dir = snapshot_download(model_id) - - vt_prefix = "model.vision_tower." - ckpt_lookup = {} - for fname in sorted(os.listdir(model_dir)): - if not fname.endswith(".safetensors"): - continue - shard_path = os.path.join(model_dir, fname) - with safe_open(shard_path, framework="pt") as f: - for key in f.keys(): - if key.startswith(vt_prefix): - model_key = key[len(vt_prefix):] - ckpt_lookup[model_key] = (shard_path, key) - - print(f" Found {len(ckpt_lookup)} vision tower keys in safetensors") - - needed = {} - for param_name in meta_params: - for candidate in [param_name, param_name.replace(".weight", ".gamma")]: - if candidate in ckpt_lookup: - shard_path, ckpt_key = ckpt_lookup[candidate] - needed.setdefault(shard_path, []).append((param_name, ckpt_key)) - break - - fixed = 0 - for shard_path, items in needed.items(): - print(f" Loading {len(items)} tensors from {os.path.basename(shard_path)}...") - with safe_open(shard_path, framework="pt", device="cpu") as f: - for param_name, ckpt_key in items: - tensor = f.get_tensor(ckpt_key) - parts = param_name.rsplit(".", 1) - parent = vision_tower - for part in parts[0].split("."): - parent = getattr(parent, part) - setattr(parent, parts[1], nn.Parameter(tensor.to(torch.float16))) - fixed += 1 - - if fixed < len(meta_params): - unmatched = [n for n in meta_params if n not in {p for items in needed.values() for p, _ in items}] - print(f" UNMATCHED ({len(unmatched)}): {unmatched[:5]}") - - print(f" Fixed {fixed}/{len(meta_params)} params") - else: - print("[2/5] All vision tower parameters on CPU - OK") - - if args.video: - combined = VisionWithProjectorVideo(vision_tower, mm_projector, LOCAL_NUM_FRAMES) - num_frames = LOCAL_NUM_FRAMES - mode_str = f"video (compress=True, T={LOCAL_NUM_FRAMES}, 16*T={16*LOCAL_NUM_FRAMES} tokens/segment)" - else: - combined = VisionWithProjectorImage(vision_tower, mm_projector) - num_frames = 1 - mode_str = "image (compress=False, 64 tokens/image)" - combined.float().eval() - print(f" Mode: {mode_str}") - - proj_params = sum(p.numel() for p in mm_projector.parameters()) - print(f" mm_projector: {proj_params:,} params, MLP {mm_projector.mm_hidden_size} → {mm_projector.mlp[0].out_features}") - - dummy_images = torch.randn(1, num_frames, 3, 224, 224) - - print("[3/5] Running a test forward pass...") - with torch.no_grad(): - test_out = combined(dummy_images) - print(f" Input: {dummy_images.shape}") - print(f" Output: {test_out.shape}") - - print("[4/5] Exporting to ONNX...") - onnx_path = "vcf-vision-video.onnx" if args.video else "vcf-vision.onnx" - data_file = onnx_path.replace(".onnx", ".onnx.data") - with torch.no_grad(): - torch.onnx.export( - combined, - (dummy_images,), - onnx_path, - input_names=["images"], - output_names=["visual_tokens"], - dynamic_axes={ - "images": {0: "batch", 1: "num_frames"}, - "visual_tokens": {0: "batch", 1: "num_visual_tokens"}, - }, - opset_version=18, - dynamo=False, - ) - - print("[5/5] Consolidating weights...") - model_proto = onnx.load(onnx_path, load_external_data=True) - - for f in os.listdir("."): - if f == onnx_path or f.endswith((".onnx.data", ".py", ".json")): - continue - if os.path.isfile(f) and not f.startswith("."): - _, ext = os.path.splitext(f) - if ext == "" or f.startswith("vision_tower") or f.startswith("mm_projector"): - os.remove(f) - - convert_model_to_external_data( - model_proto, - all_tensors_to_one_file=True, - location=data_file, - size_threshold=1024, - convert_attribute=False, - ) - onnx.save_model(model_proto, onnx_path) - - print(f"\nExported {onnx_path} + {data_file} successfully") - if args.video: - print(f" Pipeline: [{num_frames} frames] → InternVideo2 → reshape → ToMe(compress) → MLP → visual_tokens") - print(f" Fixed at T={LOCAL_NUM_FRAMES} frames (mm_local_num_frames from config)") - else: - print(" Pipeline: [1 image] → InternVideo2 → ToMe → MLP → visual_tokens") - - del combined, vision_tower, mm_projector, model_proto - gc.collect() - - -# ── Main ── -if args.embed_only: - export_embedding() -elif args.embed: - export_vision() - export_embedding() -else: - export_vision() diff --git a/examples/python/videochat-flash/run.py b/examples/python/videochat-flash/run.py deleted file mode 100644 index 96440917a6..0000000000 --- a/examples/python/videochat-flash/run.py +++ /dev/null @@ -1,119 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. -# Portions of this file consist of AI generated content. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -""" -Text-only inference test for VideoChat-Flash (OpenGVLab) exported via OGA. - -Usage: - # After exporting with builder.py --text-only: - python run.py --model ./vcf-oga-fp32-standalone - - # Custom prompt: - python run.py --model ./vcf-oga-fp32-standalone --prompt "Explain ONNX in one sentence." - -Notes: - - builder.py --text-only exports a standalone decoder with input_ids and - genai_config.json type=qwen2 (compatible with all OGA binary versions). - - Uses HF AutoTokenizer directly (og.Tokenizer may fail for this model). -""" - -import argparse -import os - -import numpy as np - -HF_MODEL_ID = "OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B" - -# Chat-ML template tokens (Qwen-style) -_IM_START = "<|im_start|>" -_IM_END = "<|im_end|>" - - -def build_prompt(user_text: str) -> str: - return f"{_IM_START}user\n{user_text}{_IM_END}\n{_IM_START}assistant\n" - - -def run_inference(model_dir: str, prompt: str, max_length: int = 256) -> str: - from onnxruntime_genai import onnxruntime_genai as og # noqa: PLC0415 - from transformers import AutoTokenizer # noqa: PLC0415 - - print(f"[1/3] Loading tokenizer from HuggingFace ({HF_MODEL_ID})...") - tok = AutoTokenizer.from_pretrained(HF_MODEL_ID, trust_remote_code=False) - - print(f"[2/3] Loading OGA model from {model_dir}...") - model = og.Model(model_dir) - - formatted = build_prompt(prompt) - input_ids = np.array(tok.encode(formatted), dtype=np.int32) - print(f"[3/3] Generating (input={len(input_ids)} tokens, max_length={max_length})...") - - params = og.GeneratorParams(model) - params.set_search_options(max_length=max_length, do_sample=False) - generator = og.Generator(model, params) - generator.append_tokens(input_ids) - - output_tokens = [] - while not generator.is_done(): - generator.generate_next_token() - tok_id = int(generator.get_next_tokens()[0]) - output_tokens.append(tok_id) - - return tok.decode(output_tokens, skip_special_tokens=True) - - -def main(): - parser = argparse.ArgumentParser( - description="Text-only inference test for VideoChat-Flash OGA export" - ) - parser.add_argument( - "--model", - type=str, - required=True, - help="Path to exported OGA model directory (e.g. ./vcf-oga-fp32-standalone)", - ) - parser.add_argument( - "--prompt", - type=str, - default="What is the capital of France? Give a short answer.", - help="User prompt (plain text, chat template applied automatically)", - ) - parser.add_argument( - "--max-length", - type=int, - default=256, - help="Maximum number of tokens to generate", - ) - parser.add_argument( - "--batch", - action="store_true", - help="Run a small batch of built-in test prompts instead of --prompt", - ) - args = parser.parse_args() - - model_dir = os.path.abspath(args.model) - - if args.batch: - test_prompts = [ - "What is the capital of France? Give a short answer.", - "What is 7 * 8?", - "Explain what ONNX Runtime is in 2 sentences.", - "Name three primary colors.", - ] - else: - test_prompts = [args.prompt] - - for i, prompt in enumerate(test_prompts, 1): - print(f"\n{'='*60}") - print(f"[{i}/{len(test_prompts)}] Prompt: {prompt}") - print("=" * 60) - response = run_inference(model_dir, prompt, args.max_length) - print(f"Response: {response}") - - print("\n[OK] Inference complete.") - - -if __name__ == "__main__": - main() diff --git a/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json b/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json deleted file mode 100644 index 8ad2c32be1..0000000000 --- a/examples/python/videochat-flash/vcf-oga-fp32/genai_config.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "model": { - "bos_token_id": 151643, - "context_length": 32768, - "decoder": { - "session_options": { - "log_id": "onnxruntime-genai", - "provider_options": [] - }, - "filename": "quantize_text.onnx", - "head_size": 128, - "hidden_size": 3584, - "inputs": { - "inputs_embeds": "inputs_embeds", - "attention_mask": "attention_mask", - "past_key_names": "past_key_values.%d.key", - "past_value_names": "past_key_values.%d.value" - }, - "outputs": { - "logits": "logits", - "present_key_names": "present.%d.key", - "present_value_names": "present.%d.value" - }, - "num_attention_heads": 28, - "num_hidden_layers": 28, - "num_key_value_heads": 4 - }, - "eos_token_id": 151645, - "pad_token_id": 151643, - "type": "videochat_flash_qwen", - "vocab_size": 152064, - "vision": { - "filename": "vcf-vision.onnx", - "config_filename": "processor_config.json", - "num_visual_tokens": 64, - "inputs": { - "pixel_values": "images" - }, - "outputs": { - "image_features": "visual_tokens" - } - }, - "embedding": { - "filename": "vcf-embed.onnx", - "inputs": { - "input_ids": "input_ids", - "image_features": "image_features" - }, - "outputs": { - "inputs_embeds": "inputs_embeds" - } - } - }, - "search": { - "diversity_penalty": 0.0, - "do_sample": false, - "early_stopping": true, - "length_penalty": 1.0, - "max_length": 32768, - "min_length": 0, - "no_repeat_ngram_size": 0, - "num_beams": 1, - "num_return_sequences": 1, - "past_present_share_buffer": true, - "repetition_penalty": 1.0, - "temperature": 1.0, - "top_k": 50, - "top_p": 1.0 - } -} \ No newline at end of file diff --git a/examples/python/videochat-flash/vcf-oga-fp32/processor_config.json b/examples/python/videochat-flash/vcf-oga-fp32/processor_config.json deleted file mode 100644 index 002eab4f92..0000000000 --- a/examples/python/videochat-flash/vcf-oga-fp32/processor_config.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "processor": { - "name": "internvideo2_image_processor", - "transforms": [ - { - "operation": { - "name": "decode_image", - "type": "DecodeImage", - "attrs": { - "color_space": "RGB" - } - } - }, - { - "operation": { - "name": "convert_to_rgb", - "type": "ConvertRGB" - } - }, - { - "operation": { - "name": "resize", - "type": "Resize", - "attrs": { - "width": 224, - "height": 224 - } - } - }, - { - "operation": { - "name": "rescale", - "type": "Rescale", - "attrs": { - "rescale_factor": 0.00392156862745098 - } - } - }, - { - "operation": { - "name": "normalize", - "type": "Normalize", - "attrs": { - "mean": [0.485, 0.456, 0.406], - "std": [0.229, 0.224, 0.225] - } - } - } - ] - } -} From 5dc8084a22f10557222238b2b70cca213d3f8c7e Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Wed, 22 Apr 2026 06:39:17 -0500 Subject: [PATCH 11/22] Revert the changes --- src/models/qwen2_5_vl_image_processor.cpp | 18 ++++++++++++++---- src/models/qwen2_5_vl_image_processor.h | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/models/qwen2_5_vl_image_processor.cpp b/src/models/qwen2_5_vl_image_processor.cpp index d3d5d2a09b..5785d1e1d5 100644 --- a/src/models/qwen2_5_vl_image_processor.cpp +++ b/src/models/qwen2_5_vl_image_processor.cpp @@ -61,7 +61,7 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr const int64_t* image_grid_thw_data = nullptr; if (pixel_values) { - // Grid-based mode: compute token count from image_grid_thw + // Get image_grid_thw data from either processor output or computed value if (image_grid_thw) { const int64_t* image_grid_thw_shape{}; size_t image_grid_thw_num_dims; @@ -73,6 +73,8 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr num_images = computed_grid_num_images; } + // Calculate total image tokens based on grid dimensions + // For each image: (temporal * height * width) / (merge_size^2) for (int64_t i = 0; i < num_images; ++i) { int64_t t = image_grid_thw_data[i * 3 + 0]; int64_t h = image_grid_thw_data[i * 3 + 1]; @@ -82,8 +84,10 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr } } + // Generate input_ids with vision tokens std::string text = prompt; + // If prompt is empty, add vision markers for each image if (text.empty()) { for (int64_t i = 0; i < num_images; ++i) { text += std::string(vision_start_token) + " " + std::string(vision_end_token); @@ -93,6 +97,8 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr } } + // Count the number of vision_start tokens and make sure it matches the number of images + // Need to escape special regex characters in the token const std::regex vision_start_regex{R"(<\|vision_start\|>)"}; const auto vision_start_begin = std::sregex_iterator(text.begin(), text.end(), vision_start_regex); const auto vision_start_end = std::sregex_iterator(); @@ -103,8 +109,8 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr " vision_start tokens but received " + std::to_string(num_images) + " images."); } - // Replace vision markers with the correct number of image_pad tokens per image. - // The count is derived from (T*H*W) / spatial_merge_size^2. + // For Qwen2-VL, we need to replace vision markers with image_pad tokens + // The number of image_pad tokens for each image depends on the image dimensions if (num_images > 0 && image_grid_thw_data) { std::string modified_text; size_t last_pos = 0; @@ -113,13 +119,16 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr std::smatch match; std::string temp_text = text; while (std::regex_search(temp_text, match, vision_start_regex)) { + // Add text before the vision_start token modified_text += text.substr(last_pos, match.position() - (last_pos - (text.size() - temp_text.size()))); + // Calculate number of image_pad tokens for this image int64_t t = image_grid_thw_data[image_idx * 3 + 0]; int64_t h = image_grid_thw_data[image_idx * 3 + 1]; int64_t w = image_grid_thw_data[image_idx * 3 + 2]; int64_t num_pads = (t * h * w) / (spatial_merge_size * spatial_merge_size); + // Add vision_start, image_pad tokens, and vision_end modified_text += vision_start_token; for (int64_t i = 0; i < num_pads; ++i) { modified_text += image_pad_token; @@ -128,6 +137,7 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr last_pos = match.position() + match.length() + (text.size() - temp_text.size()); + // Find and skip vision_end token size_t vision_end_pos = text.find(vision_end_token, last_pos); if (vision_end_pos != std::string::npos) { last_pos = vision_end_pos + strlen(vision_end_token); @@ -156,7 +166,7 @@ ProcessImagePrompt(const Generators::Tokenizer& tokenizer, const std::string& pr } // namespace QwenImageProcessor::QwenImageProcessor(Config& config, const SessionInfo& session_info) - : pixel_values_type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT}, + : pixel_values_type_{ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT}, // Default to float, will be determined at runtime if vision session exists spatial_merge_size_{config.model.vision.spatial_merge_size}, patch_size_{config.model.vision.patch_size} { const auto processor_config = (config.config_path / fs::path(config.model.vision.config_filename)).string(); diff --git a/src/models/qwen2_5_vl_image_processor.h b/src/models/qwen2_5_vl_image_processor.h index e72bc0f018..8dfb78bc1e 100644 --- a/src/models/qwen2_5_vl_image_processor.h +++ b/src/models/qwen2_5_vl_image_processor.h @@ -19,7 +19,7 @@ struct QwenImageProcessor : Processor { ONNXTensorElementDataType pixel_values_type_; int64_t spatial_merge_size_; - int64_t patch_size_{14}; + int64_t patch_size_{14}; // Qwen2.5-VL uses 14, Qwen3-VL uses 16 }; } // namespace Generators From f7e7b5d7f88016f07ebbe32db786b4cb5fbcc269 Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Wed, 22 Apr 2026 06:53:49 -0500 Subject: [PATCH 12/22] Add license --- src/config.cpp | 3 ++- src/config.h | 3 ++- src/models/model.cpp | 3 ++- src/models/model.h | 3 ++- src/models/videochat_flash_processor.cpp | 6 ++++-- src/models/videochat_flash_processor.h | 6 ++++-- src/python/py/models/builder.py | 3 ++- src/python/py/models/builders/__init__.py | 2 +- src/python/py/models/builders/qwen.py | 3 ++- 9 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index 59487f6394..c86c5a280e 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// Modifications Copyright(C) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #include "generators.h" #include "models/model_type.h" #include "runtime_settings.h" diff --git a/src/config.h b/src/config.h index 281d6f4ad9..08cf89a294 100644 --- a/src/config.h +++ b/src/config.h @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// Modifications Copyright(C) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #pragma once namespace Generators { diff --git a/src/models/model.cpp b/src/models/model.cpp index e256c67742..5d90d45345 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // -// Modifications Copyright(C) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #include #include #include diff --git a/src/models/model.h b/src/models/model.h index b3e5583362..a9751c5a6c 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -1,7 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // -// Modifications Copyright(C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. #pragma once #include "model_type.h" #include "ortx_tokenizer.h" diff --git a/src/models/videochat_flash_processor.cpp b/src/models/videochat_flash_processor.cpp index 65ac59d1f9..b9951c2a94 100644 --- a/src/models/videochat_flash_processor.cpp +++ b/src/models/videochat_flash_processor.cpp @@ -1,5 +1,7 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. +// Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. +// Licensed under the MIT License. See License.txt in the project root for +// license information. #include "../generators.h" #include "model.h" diff --git a/src/models/videochat_flash_processor.h b/src/models/videochat_flash_processor.h index b40576602d..320dbe96a7 100644 --- a/src/models/videochat_flash_processor.h +++ b/src/models/videochat_flash_processor.h @@ -1,5 +1,7 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. +// Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +// Portions of this file consist of AI generated content. +// Licensed under the MIT License. See License.txt in the project root for +// license information. #pragma once diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index 0d623a0b39..4c251b0770 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -3,7 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. Portions of this file consist of AI generated content. +# Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# Portions of this file consist of AI generated content. # -------------------------------------------------------------------------- """ Run the model builder to create the desired ONNX model. diff --git a/src/python/py/models/builders/__init__.py b/src/python/py/models/builders/__init__.py index 2ec9e7667e..60ae0c8235 100644 --- a/src/python/py/models/builders/__init__.py +++ b/src/python/py/models/builders/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # ------------------------------------------------------------------------- -# Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. +# Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. # Portions of this file consist of AI generated content. # ------------------------------------------------------------------------- from .base import Model diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index d84cdef9ab..260a6ac484 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -3,7 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- - +# Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. All rights reserved. +# Portions of this file consist of AI generated content. import os From 4493383d9e083db3c653241896c7c02cc9dd409d Mon Sep 17 00:00:00 2001 From: "Jain, Vishal" Date: Thu, 23 Apr 2026 15:46:33 +0530 Subject: [PATCH 13/22] Add spdx-mit --- src/models/videochat_flash_processor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/models/videochat_flash_processor.cpp b/src/models/videochat_flash_processor.cpp index b9951c2a94..6c9b2c3d19 100644 --- a/src/models/videochat_flash_processor.cpp +++ b/src/models/videochat_flash_processor.cpp @@ -1,5 +1,8 @@ // Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. // Portions of this file consist of AI generated content. +// +// SPDX-License-Identifier: MIT +// // Licensed under the MIT License. See License.txt in the project root for // license information. From fcf7278c233be946df36befccaa6a4cbe19c0869 Mon Sep 17 00:00:00 2001 From: "Jain, Vishal" Date: Thu, 23 Apr 2026 15:47:01 +0530 Subject: [PATCH 14/22] Add spdx-mit --- src/models/videochat_flash_processor.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/models/videochat_flash_processor.h b/src/models/videochat_flash_processor.h index 320dbe96a7..75ca17e492 100644 --- a/src/models/videochat_flash_processor.h +++ b/src/models/videochat_flash_processor.h @@ -1,5 +1,8 @@ // Copyright (C) [2026] Advanced Micro Devices, Inc. All rights reserved. // Portions of this file consist of AI generated content. +// +// SPDX-License-Identifier: MIT +// // Licensed under the MIT License. See License.txt in the project root for // license information. From 47ebb9e293b02be922e9c5ad1ca5d95c0cc726ee Mon Sep 17 00:00:00 2001 From: anilmartha Date: Mon, 18 May 2026 11:03:08 +0530 Subject: [PATCH 15/22] Apply review changes Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/python/py/models/builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/py/models/builders/__init__.py b/src/python/py/models/builders/__init__.py index 62cc4cb874..3f26abb790 100644 --- a/src/python/py/models/builders/__init__.py +++ b/src/python/py/models/builders/__init__.py @@ -28,7 +28,7 @@ Phi4MMModel, PhiModel, ) -from .qwen import Qwen3Model, Qwen25VLTextModel, Qwen3VLTextModel, QwenModel, VideoChatFlashQwenModel +from .qwen import Qwen3Model, Qwen25VLTextModel, Qwen3VLTextModel, Qwen35TextModel, QwenModel, VideoChatFlashQwenModel from .smollm import SmolLM3Model from .whisper import WhisperModel From c1717b8028f162242166033ef0014ff4db7bb1da Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Mon, 18 May 2026 01:07:07 -0500 Subject: [PATCH 16/22] Address PR #2147 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - builder.py: introduce _has_vcf_architecture() helper that handles architectures as a list, a bare string, or absent/unknown type, then use it in both the local-dir and HF-hub code paths (Copilot comment #2) - config.h: correct misleading comment on num_visual_tokens — remove the false "0 = compute from image_grid_thw" claim; the field must be > 0 for videochat_flash_qwen (Copilot comment #3) - videochat_flash_processor.cpp: add clarifying comment to empty catch block — exception is expected when only the language decoder session is loaded (Copilot comment #4) - builders/__init__.py: move VideoChatFlashQwenModel to its correct alphabetical position in __all__ (after SmolLM3Model, before WhisperModel) per kunal-vaishnavi comment #6 Co-Authored-By: Claude Sonnet 4 --- src/config.h | 2 +- src/models/videochat_flash_processor.cpp | 2 ++ src/python/py/models/builder.py | 13 +++++++++++-- src/python/py/models/builders/__init__.py | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/config.h b/src/config.h index 07e1d62b9a..1cd6647354 100644 --- a/src/config.h +++ b/src/config.h @@ -219,7 +219,7 @@ struct Config { // and these values are unused. int spatial_merge_size{2}; float tokens_per_second{2.0f}; - int num_visual_tokens{0}; // Fixed visual tokens per image; 0 = compute from image_grid_thw + int num_visual_tokens{0}; // Fixed visual tokens per image; must be > 0 for videochat_flash_qwen int patch_size{14}; // Qwen2.5-VL uses 14, Qwen3-VL uses 16 int window_size{0}; // Used by CalculateWindowIndex() in QNN pipeline only. // 0 = auto-compute as patch_size * spatial_merge_size * 2 diff --git a/src/models/videochat_flash_processor.cpp b/src/models/videochat_flash_processor.cpp index 6c9b2c3d19..befe2bac85 100644 --- a/src/models/videochat_flash_processor.cpp +++ b/src/models/videochat_flash_processor.cpp @@ -121,6 +121,8 @@ VideoChatFlashProcessor::VideoChatFlashProcessor(Config& config, const SessionIn try { pixel_values_type_ = session_info.GetInputDataType(config.model.vision.inputs.pixel_values); } catch (...) { + // pixel_values input may be absent when only the language decoder session is loaded; + // the default-initialized pixel_values_type_ (FLOAT) is used in that case. } config.AddMapping(std::string(Config::Defaults::InputIdsName), config.model.embedding.inputs.input_ids); diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index 1c10da2f5f..476a787c68 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -197,6 +197,15 @@ def create_model( # (av, cv2, decord, imageio) even though the LM backbone is standard Qwen2.5. # Load the raw config.json via Qwen2Config to avoid pulling in video deps. _vcf_arch = "VideoChatFlashQwenForCausalLM" + + def _has_vcf_architecture(raw_config: dict[str, Any]) -> bool: + architectures = raw_config.get("architectures") + if isinstance(architectures, list): + return _vcf_arch in architectures + if isinstance(architectures, str): + return architectures == _vcf_arch + return False + _is_vcf = False try: import json as _json @@ -204,13 +213,13 @@ def create_model( _raw_cfg_path = os.path.join(hf_name, "config.json") if os.path.isfile(_raw_cfg_path): with open(_raw_cfg_path) as _f: - _is_vcf = _json.load(_f).get("architectures", [None])[0] == _vcf_arch + _is_vcf = _has_vcf_architecture(_json.load(_f)) else: # HF repo: peek at config.json without running custom code from huggingface_hub import hf_hub_download _cfg_file = hf_hub_download(repo_id=hf_name, filename="config.json", token=hf_token, cache_dir=cache_dir) with open(_cfg_file) as _f: - _is_vcf = _json.load(_f).get("architectures", [None])[0] == _vcf_arch + _is_vcf = _has_vcf_architecture(_json.load(_f)) except Exception: pass diff --git a/src/python/py/models/builders/__init__.py b/src/python/py/models/builders/__init__.py index 3f26abb790..3522467dda 100644 --- a/src/python/py/models/builders/__init__.py +++ b/src/python/py/models/builders/__init__.py @@ -61,7 +61,7 @@ "Qwen25VLTextModel", "Qwen35TextModel", "QwenModel", - "VideoChatFlashQwenModel", "SmolLM3Model", + "VideoChatFlashQwenModel", "WhisperModel", ] From 9315ab7b812715872f8d053bf7e3094630b49a8c Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Mon, 18 May 2026 05:23:28 -0500 Subject: [PATCH 17/22] Address CodeQL findings --- src/python/py/models/builder.py | 5 +++++ src/python/py/models/builders/qwen.py | 23 ++++++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index 476a787c68..a45303a541 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -221,6 +221,11 @@ def _has_vcf_architecture(raw_config: dict[str, Any]) -> bool: with open(_cfg_file) as _f: _is_vcf = _has_vcf_architecture(_json.load(_f)) except Exception: + # Best-effort architecture peek: if config.json is unreadable, the + # huggingface_hub download fails (offline, private repo without token, + # missing file), or the JSON is malformed, fall through to the standard + # AutoConfig.from_pretrained() path below, which has richer error + # reporting and is the canonical loader for non-VCF models. pass if _is_vcf: diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 5b84539740..5cfa596f09 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -926,12 +926,25 @@ class VideoChatFlashQwenModel(QwenModel): """ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): + # Pre-configure before super().__init__() so the base class (Model) + # establishes these attributes on first assignment instead of us + # overwriting inherited values. + # + # The custom remote code requires video libraries (av, cv2, decord, + # imageio) which are not needed to export the LM backbone. Force + # hf_remote=False so base class helpers (make_genai_config, + # save_processing) use standard, non-remote-code paths. + extra_options = dict(extra_options) if extra_options else {} + extra_options["hf_remote"] = False + + # Model.__init__ sets self.model_type = config.architectures[0]. The + # HF architecture string ("VideoChatFlashQwenForCausalLM") has already + # served its purpose in the dispatch table in builder.py; swap it for + # the runtime identifier expected by genai_config.json and the C++ + # runtime registration in model.cpp. + config.architectures = ["videochat_flash_qwen"] + super().__init__(config, io_dtype, onnx_dtype, ep, cache_dir, extra_options) - self.model_type = "videochat_flash_qwen" - # The custom remote code requires video libraries (av, cv2, decord, imageio) - # which are not needed to export the LM backbone. Disable trust_remote_code - # so base class helpers (make_genai_config, save_processing) use standard paths. - self.hf_remote = False def make_genai_config(self, model_name_or_path, extra_kwargs, out_dir): # make_genai_config in base.py calls AutoConfig with trust_remote_code, From 4faa201b7f6a2e14285ff827d4206096cc28a53a Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Tue, 19 May 2026 04:09:42 -0500 Subject: [PATCH 18/22] Fix linter --- src/config.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config.h b/src/config.h index 1cd6647354..e2e4f6e869 100644 --- a/src/config.h +++ b/src/config.h @@ -220,10 +220,10 @@ struct Config { int spatial_merge_size{2}; float tokens_per_second{2.0f}; int num_visual_tokens{0}; // Fixed visual tokens per image; must be > 0 for videochat_flash_qwen - int patch_size{14}; // Qwen2.5-VL uses 14, Qwen3-VL uses 16 - int window_size{0}; // Used by CalculateWindowIndex() in QNN pipeline only. - // 0 = auto-compute as patch_size * spatial_merge_size * 2 - // Qwen2.5-VL default: 56 (14*4), Qwen3-VL default: 64 (16*4) + int patch_size{14}; // Qwen2.5-VL uses 14, Qwen3-VL uses 16 + int window_size{0}; // Used by CalculateWindowIndex() in QNN pipeline only. + // 0 = auto-compute as patch_size * spatial_merge_size * 2 + // Qwen2.5-VL default: 56 (14*4), Qwen3-VL default: 64 (16*4) std::string config_filename{"processor_config.json"}; std::optional adapter_filename{}; From 23d25ea80b0e2aa594dcd5bc38787b42c3ea4fcf Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Wed, 20 May 2026 13:16:04 -0500 Subject: [PATCH 19/22] Address review comments --- src/models/videochat_flash_processor.cpp | 108 +++++++++++------------ src/python/py/models/builder.py | 8 +- src/python/py/models/builders/qwen.py | 54 ++---------- 3 files changed, 59 insertions(+), 111 deletions(-) diff --git a/src/models/videochat_flash_processor.cpp b/src/models/videochat_flash_processor.cpp index befe2bac85..1af9bfe303 100644 --- a/src/models/videochat_flash_processor.cpp +++ b/src/models/videochat_flash_processor.cpp @@ -15,36 +15,6 @@ namespace Generators { namespace { -std::unique_ptr ConvertPixelValues(const OrtValue& float_tensor, - ONNXTensorElementDataType target_type, - Ort::Allocator& allocator) { - auto shape = float_tensor.GetTensorTypeAndShapeInfo()->GetShape(); - size_t count = float_tensor.GetTensorTypeAndShapeInfo()->GetElementCount(); - - if (target_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { - auto result = OrtValue::CreateTensor(allocator, shape); - std::copy(float_tensor.GetTensorData(), - float_tensor.GetTensorData() + count, - result->GetTensorMutableData()); - return result; - } - - std::unique_ptr result; - if (target_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16) { - result = OrtValue::CreateTensor(allocator, shape); - } else if (target_type == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) { - result = OrtValue::CreateTensor(allocator, shape); - } else { - throw std::runtime_error("Unsupported target type for pixel values conversion"); - } - - auto* cpu_device = GetDeviceInterface(DeviceType::CPU); - void* input_data = const_cast(static_cast(float_tensor.GetTensorData())); - void* output_data = result->GetTensorMutableRawData(); - cpu_device->Cast(input_data, output_data, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, target_type, count); - return result; -} - // Build input_ids from prompt, inserting fixed_tokens_per_image <|image_pad|> tokens per image. std::tuple, std::unique_ptr> BuildPromptTokens(const Tokenizer& tokenizer, const std::string& prompt, @@ -149,8 +119,9 @@ std::unique_ptr VideoChatFlashProcessor::Process(const Tokenizer& ort_extensions::OrtxObjectPtr result; CheckResult(OrtxImagePreProcess(processor_.get(), images->images_.get(), result.ToBeAssigned())); - OrtxTensor* pixel_values = nullptr; - CheckResult(OrtxTensorResultGetAt(result.get(), 0, &pixel_values)); + ort_extensions::OrtxObjectPtr pixel_values_owner; + CheckResult(OrtxTensorResultGetAt(result.get(), 0, pixel_values_owner.ToBeAssigned())); + OrtxTensor* pixel_values = pixel_values_owner.get(); const float* pv_data{}; const int64_t* pv_shape{}; @@ -158,43 +129,64 @@ std::unique_ptr VideoChatFlashProcessor::Process(const Tokenizer& CheckResult(OrtxGetTensorData(pixel_values, reinterpret_cast(&pv_data), &pv_shape, &pv_ndims)); - // Determine layout from ORT Extensions output (HWC format) - int64_t num_imgs, height, width, channels; + // Detect whether ORT Extensions output is HWC or CHW. + // Once processor_config.json includes a Permute3D step, the output will be + // NCHW and the HWC path below can be removed. + int64_t num_imgs, channels, height, width; + bool is_hwc; if (pv_ndims == 3) { num_imgs = 1; - height = pv_shape[0]; - width = pv_shape[1]; - channels = pv_shape[2]; + // CHW: [C, H, W] vs HWC: [H, W, C] — channel dim is the small one + is_hwc = (pv_shape[2] < pv_shape[0]); + if (is_hwc) { + height = pv_shape[0]; width = pv_shape[1]; channels = pv_shape[2]; + } else { + channels = pv_shape[0]; height = pv_shape[1]; width = pv_shape[2]; + } } else if (pv_ndims == 4) { num_imgs = pv_shape[0]; - height = pv_shape[1]; - width = pv_shape[2]; - channels = pv_shape[3]; + is_hwc = (pv_shape[3] < pv_shape[1]); + if (is_hwc) { + height = pv_shape[1]; width = pv_shape[2]; channels = pv_shape[3]; + } else { + channels = pv_shape[1]; height = pv_shape[2]; width = pv_shape[3]; + } } else { throw std::runtime_error("VideoChatFlashProcessor: unexpected pixel_values rank " + std::to_string(pv_ndims) + " (expected 3 or 4)"); } - // Transpose HWC → CHW and reshape to [1, num_frames, C, H, W] - std::vector target_shape = {1, num_imgs, channels, height, width}; - auto float_tensor = OrtValue::CreateTensor(allocator, target_shape); - float* dst = float_tensor->GetTensorMutableData(); - - for (int64_t n = 0; n < num_imgs; ++n) { - const float* src_img = pv_data + n * height * width * channels; - float* dst_img = dst + n * channels * height * width; - for (int64_t c = 0; c < channels; ++c) { - for (int64_t h = 0; h < height; ++h) { - for (int64_t w = 0; w < width; ++w) { - dst_img[c * height * width + h * width + w] = src_img[h * width * channels + w * channels + c]; - } + // Vision model expects [1, num_frames, C, H, W] + { + std::vector target_shape = {1, num_imgs, channels, height, width}; + size_t count = static_cast(num_imgs * channels * height * width); + + auto float_tensor = OrtValue::CreateTensor(allocator, target_shape); + float* dst = float_tensor->GetTensorMutableData(); + + if (is_hwc) { + for (int64_t n = 0; n < num_imgs; ++n) { + const float* src_img = pv_data + n * height * width * channels; + float* dst_img = dst + n * channels * height * width; + for (int64_t c = 0; c < channels; ++c) + for (int64_t h = 0; h < height; ++h) + for (int64_t w = 0; w < width; ++w) + dst_img[c * height * width + h * width + w] = src_img[h * width * channels + w * channels + c]; } + } else { + std::copy(pv_data, pv_data + count, dst); } - } - auto converted_pv = ConvertPixelValues(*float_tensor, pixel_values_type_, allocator); - named_tensors->emplace(std::string(Config::Defaults::PixelValuesName), - std::make_shared(std::move(converted_pv))); + std::unique_ptr pv_ortvalue; + if (pixel_values_type_ == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) { + pv_ortvalue = std::move(float_tensor); + } else { + auto* p_device = GetDeviceInterface(DeviceType::CPU); + Cast(*float_tensor, pv_ortvalue, *p_device, pixel_values_type_); + } + named_tensors->emplace(std::string(Config::Defaults::PixelValuesName), + std::make_shared(std::move(pv_ortvalue))); + } // Tokenize prompt with fixed visual token padding auto [input_ids, num_img_tokens] = BuildPromptTokens( @@ -216,7 +208,7 @@ std::unique_ptr VideoChatFlashProcessor::Process(const Tokenizer& grid_ptr[i * 3 + 1] = height; grid_ptr[i * 3 + 2] = width; } - named_tensors->emplace("image_grid_thw", + named_tensors->emplace(std::string(Config::Defaults::ImageGridThwName), std::make_shared(std::move(grid_thw))); return named_tensors; diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index a45303a541..dd72a36cb2 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -46,8 +46,8 @@ Qwen25VLTextModel, Qwen35TextModel, QwenModel, - VideoChatFlashQwenModel, SmolLM3Model, + VideoChatFlashQwenModel, WhisperModel, ) from transformers import ( @@ -333,9 +333,9 @@ def _has_vcf_architecture(raw_config: dict[str, Any]) -> bool: elif config.architectures[0] == "Qwen2ForCausalLM": onnx_model = QwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "VideoChatFlashQwenForCausalLM": - if "exclude_embeds" not in extra_options: - print("WARNING: This is only generating the text component of the model. Setting `--extra_options exclude_embeds=true` by default.") - extra_options["exclude_embeds"] = True + print("WARNING: This is only generating the text component of the model. Setting `--extra_options exclude_embeds=true` by default.") + extra_options["exclude_embeds"] = True + extra_options["hf_remote"] = False onnx_model = VideoChatFlashQwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "Qwen2_5_VLForConditionalGeneration": text_config = config.text_config diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 17e7fafe2b..232c62a8b2 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -930,57 +930,13 @@ class VideoChatFlashQwenModel(QwenModel): """ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): - # Pre-configure before super().__init__() so the base class (Model) - # establishes these attributes on first assignment instead of us - # overwriting inherited values. - # - # The custom remote code requires video libraries (av, cv2, decord, - # imageio) which are not needed to export the LM backbone. Force - # hf_remote=False so base class helpers (make_genai_config, - # save_processing) use standard, non-remote-code paths. - extra_options = dict(extra_options) if extra_options else {} - extra_options["hf_remote"] = False - - # Model.__init__ sets self.model_type = config.architectures[0]. The - # HF architecture string ("VideoChatFlashQwenForCausalLM") has already - # served its purpose in the dispatch table in builder.py; swap it for - # the runtime identifier expected by genai_config.json and the C++ - # runtime registration in model.cpp. - config.architectures = ["videochat_flash_qwen"] - super().__init__(config, io_dtype, onnx_dtype, ep, cache_dir, extra_options) - def make_genai_config(self, model_name_or_path, extra_kwargs, out_dir): - # make_genai_config in base.py calls AutoConfig with trust_remote_code, - # which triggers the video library imports. Instead, write a clean - # Qwen2-compatible config.json to a temp dir and let base class read it. - import json as _json - import shutil - import tempfile - - from transformers import Qwen2Config - - vcf_config = Qwen2Config.from_pretrained(model_name_or_path, token=self.hf_token, **extra_kwargs) - vcf_config.architectures = ["VideoChatFlashQwenForCausalLM"] - vcf_config.model_type = "videochat_flash_qwen" - vcf_config._name_or_path = model_name_or_path - - tmp_dir = tempfile.mkdtemp() - try: - with open(os.path.join(tmp_dir, "config.json"), "w") as f: - _json.dump(vcf_config.to_dict(), f) - super().make_genai_config(tmp_dir, {}, out_dir) - finally: - shutil.rmtree(tmp_dir, ignore_errors=True) - - # Restore the correct model type (base class may write "qwen2" from Qwen2Config) - gcfg_path = os.path.join(out_dir, "genai_config.json") - if os.path.isfile(gcfg_path): - with open(gcfg_path) as f: - gcfg = _json.load(f) - gcfg["model"]["type"] = "videochat_flash_qwen" - with open(gcfg_path, "w") as f: - _json.dump(gcfg, f, indent=2) + # Override model_type for the C++ runtime registration in model.cpp + # and genai_config.json. Same pattern as Qwen3VLTextModel. + # Base class transforms this to "videochat_flash_qwen" via: + # model_type[:model_type.find("For")].lower() + self.model_type = "VideoChat_Flash_QwenForCausalLM" def load_weights(self, input_path): # The LM backbone is identical to Qwen2ForCausalLM. Load it directly From bd1cf339c98321f321477c5233ed8bd603fc5d2a Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Thu, 21 May 2026 04:40:01 -0500 Subject: [PATCH 20/22] Remove VCF-specific config bypass and hf_remote override in builder.py Address PR #2147 review: keep the config loading section generic by removing the VideoChat-Flash architecture peek/Qwen2Config workaround. Use the standard AutoConfig.from_pretrained path for all models. Also remove the forced hf_remote=False in the VCF dispatch branch so HF model dependencies (video libraries) are loaded normally, consistent with how other models handle their library requirements. Co-authored-by: Cursor --- src/python/py/models/builder.py | 44 +-------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/src/python/py/models/builder.py b/src/python/py/models/builder.py index dd72a36cb2..fb1a65eae8 100644 --- a/src/python/py/models/builder.py +++ b/src/python/py/models/builder.py @@ -193,48 +193,7 @@ def create_model( hf_token = parse_hf_token(extra_options.get("hf_token", "true")) hf_remote = extra_options.get("hf_remote", True) - # VideoChat-Flash uses custom remote code that imports heavy video libraries - # (av, cv2, decord, imageio) even though the LM backbone is standard Qwen2.5. - # Load the raw config.json via Qwen2Config to avoid pulling in video deps. - _vcf_arch = "VideoChatFlashQwenForCausalLM" - - def _has_vcf_architecture(raw_config: dict[str, Any]) -> bool: - architectures = raw_config.get("architectures") - if isinstance(architectures, list): - return _vcf_arch in architectures - if isinstance(architectures, str): - return architectures == _vcf_arch - return False - - _is_vcf = False - try: - import json as _json - if os.path.isdir(hf_name): - _raw_cfg_path = os.path.join(hf_name, "config.json") - if os.path.isfile(_raw_cfg_path): - with open(_raw_cfg_path) as _f: - _is_vcf = _has_vcf_architecture(_json.load(_f)) - else: - # HF repo: peek at config.json without running custom code - from huggingface_hub import hf_hub_download - _cfg_file = hf_hub_download(repo_id=hf_name, filename="config.json", token=hf_token, cache_dir=cache_dir) - with open(_cfg_file) as _f: - _is_vcf = _has_vcf_architecture(_json.load(_f)) - except Exception: - # Best-effort architecture peek: if config.json is unreadable, the - # huggingface_hub download fails (offline, private repo without token, - # missing file), or the JSON is malformed, fall through to the standard - # AutoConfig.from_pretrained() path below, which has richer error - # reporting and is the canonical loader for non-VCF models. - pass - - if _is_vcf: - from transformers import Qwen2Config - config = Qwen2Config.from_pretrained(hf_name, token=hf_token, **extra_kwargs) - config.architectures = [_vcf_arch] - config._name_or_path = hf_name # ensure load_weights can find the weights - else: - config = AutoConfig.from_pretrained(hf_name, token=hf_token, trust_remote_code=hf_remote, **extra_kwargs) + config = AutoConfig.from_pretrained(hf_name, token=hf_token, trust_remote_code=hf_remote, **extra_kwargs) if "adapter_path" in extra_options: from peft import PeftConfig @@ -335,7 +294,6 @@ def _has_vcf_architecture(raw_config: dict[str, Any]) -> bool: elif config.architectures[0] == "VideoChatFlashQwenForCausalLM": print("WARNING: This is only generating the text component of the model. Setting `--extra_options exclude_embeds=true` by default.") extra_options["exclude_embeds"] = True - extra_options["hf_remote"] = False onnx_model = VideoChatFlashQwenModel(config, io_dtype, onnx_dtype, execution_provider, cache_dir, extra_options) elif config.architectures[0] == "Qwen2_5_VLForConditionalGeneration": text_config = config.text_config From cb226b764a6d16414f031f1d5286460c5ef18f93 Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Thu, 21 May 2026 04:57:30 -0500 Subject: [PATCH 21/22] Fix linter --- src/models/videochat_flash_processor.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/models/videochat_flash_processor.cpp b/src/models/videochat_flash_processor.cpp index 1af9bfe303..2f2c57c108 100644 --- a/src/models/videochat_flash_processor.cpp +++ b/src/models/videochat_flash_processor.cpp @@ -139,17 +139,25 @@ std::unique_ptr VideoChatFlashProcessor::Process(const Tokenizer& // CHW: [C, H, W] vs HWC: [H, W, C] — channel dim is the small one is_hwc = (pv_shape[2] < pv_shape[0]); if (is_hwc) { - height = pv_shape[0]; width = pv_shape[1]; channels = pv_shape[2]; + height = pv_shape[0]; + width = pv_shape[1]; + channels = pv_shape[2]; } else { - channels = pv_shape[0]; height = pv_shape[1]; width = pv_shape[2]; + channels = pv_shape[0]; + height = pv_shape[1]; + width = pv_shape[2]; } } else if (pv_ndims == 4) { num_imgs = pv_shape[0]; is_hwc = (pv_shape[3] < pv_shape[1]); if (is_hwc) { - height = pv_shape[1]; width = pv_shape[2]; channels = pv_shape[3]; + height = pv_shape[1]; + width = pv_shape[2]; + channels = pv_shape[3]; } else { - channels = pv_shape[1]; height = pv_shape[2]; width = pv_shape[3]; + channels = pv_shape[1]; + height = pv_shape[2]; + width = pv_shape[3]; } } else { throw std::runtime_error("VideoChatFlashProcessor: unexpected pixel_values rank " + From f9dab303ebc6878a36b713374d0a70f6369d2ae9 Mon Sep 17 00:00:00 2001 From: Anil Kumar Martha Date: Fri, 22 May 2026 04:35:36 -0500 Subject: [PATCH 22/22] Move Qwen2ForCausalLM import to module-level in qwen.py --- src/python/py/models/builders/qwen.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/python/py/models/builders/qwen.py b/src/python/py/models/builders/qwen.py index 232c62a8b2..2d7adf163b 100644 --- a/src/python/py/models/builders/qwen.py +++ b/src/python/py/models/builders/qwen.py @@ -13,6 +13,7 @@ import torch from transformers import ( AutoConfig, + Qwen2ForCausalLM, Qwen2_5_VLForConditionalGeneration, Qwen3VLForConditionalGeneration, ) @@ -939,10 +940,6 @@ def __init__(self, config, io_dtype, onnx_dtype, ep, cache_dir, extra_options): self.model_type = "VideoChat_Flash_QwenForCausalLM" def load_weights(self, input_path): - # The LM backbone is identical to Qwen2ForCausalLM. Load it directly - # to avoid the custom remote code (which requires video libraries). - from transformers import Qwen2ForCausalLM - print("Loading VideoChatFlash model as Qwen2ForCausalLM...") extra_kwargs = {} if os.path.isdir(self.model_name_or_path) else {"cache_dir": self.cache_dir} return Qwen2ForCausalLM.from_pretrained( self.model_name_or_path,