From 42717823c6916e45b18cbad19d5f036be4884566 Mon Sep 17 00:00:00 2001 From: titaiwang Date: Fri, 8 May 2026 16:40:04 +0000 Subject: [PATCH 1/2] Fix DecoderState input_ids check regression introduced in #2103 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #2103 (Gemma4 multimodal support) added an optional decoder_input_ids_ for models requiring input_ids alongside inputs_embeds (e.g. Gemma4). However, the check used the combined model_.session_info_ which aggregates inputs from ALL sessions (decoder + embedding + vision + speech). Because the embedding session always has input_ids as its primary input, HasInput('input_ids') returns true for every VLM with an embedding model — including mistral3, whose decoder ONNX only accepts inputs_embeds. This caused input_ids to be injected into the decoder's ORT feed at runtime, producing: RuntimeError: Invalid input name: input_ids Fix: create a decoder-only SessionInfo for this check so it only fires when the decoder ONNX itself actually declares input_ids as an input. Gemma4 behaviour is preserved since its decoder ONNX has input_ids. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/models/multi_modal.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/models/multi_modal.cpp b/src/models/multi_modal.cpp index 6a628168bd..541377646a 100644 --- a/src/models/multi_modal.cpp +++ b/src/models/multi_modal.cpp @@ -650,10 +650,17 @@ DecoderState::DecoderState(const MultiModalLanguageModel& model, DeviceSpanmodel.decoder.inputs.input_ids)) { - decoder_input_ids_ = std::make_unique(*this); - decoder_input_ids_->Add(); + // Some multimodal decoders (e.g., Gemma4) require input_ids alongside inputs_embeds. + // Use a decoder-only SessionInfo to avoid false positives: the combined session_info_ + // includes embedding session inputs (which always has input_ids), causing this check + // to incorrectly fire for models like mistral3 whose decoder has no input_ids input. + { + SessionInfo decoder_only_info; + decoder_only_info.Add(*model_.decoder_session_); + if (decoder_only_info.HasInput(model_.config_->model.decoder.inputs.input_ids)) { + decoder_input_ids_ = std::make_unique(*this); + decoder_input_ids_->Add(); + } } position_inputs_->Add(); From db445482b69f7dc80965b3cb76a0d7fe0529d8fd Mon Sep 17 00:00:00 2001 From: titaiwang Date: Fri, 8 May 2026 22:21:09 +0000 Subject: [PATCH 2/2] test: add Python test for DecoderState input_ids injection fix (PR #2148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bug: DecoderState constructor used combined session_info_ (decoder + vision + embedding sessions) to check HasInput('input_ids'). The embedding session always declares input_ids, so the check incorrectly injected input_ids into the decoder feed for models like Mistral3 whose decoder has no input_ids input — causing an ORT 'Invalid Feed Input Name' error. The fix (src/models/multi_modal.cpp, commit 4271782): use a decoder-only SessionInfo for the HasInput('input_ids') check. Two test model variants are added to test/test_models/: - multimodal-decoder-no-input-ids/ (Mistral3-like: embedding has input_ids, decoder does NOT — the case that was broken) - multimodal-decoder-with-input-ids/ (Gemma4-like: both embedding and decoder declare input_ids — should succeed with or without fix) test/python/test_decoder_state_input_ids.py exercises both: - Mistral3-like: generation must succeed; with the pre-fix code it would fail because input_ids would be incorrectly fed to a decoder session that never declared it. - Gemma4-like: generation must also succeed; the fix correctly identifies that the decoder declares input_ids and feeds it. test/test_models/create_decoder_input_ids_test_models.py is the script used to regenerate the dummy ONNX model files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 5 + test/python/test_decoder_state_input_ids.py | 82 +++++ .../create_decoder_input_ids_test_models.py | 348 ++++++++++++++++++ .../dummy_embedding.onnx | Bin 0 -> 433 bytes .../dummy_text.onnx | Bin 0 -> 3166 bytes .../dummy_vision.onnx | Bin 0 -> 462 bytes .../genai_config.json | 62 ++++ .../tokenizer.json | 51 +++ .../tokenizer_config.json | 7 + .../dummy_embedding.onnx | Bin 0 -> 433 bytes .../dummy_text.onnx | Bin 0 -> 3203 bytes .../dummy_vision.onnx | Bin 0 -> 462 bytes .../genai_config.json | 62 ++++ .../tokenizer.json | 51 +++ .../tokenizer_config.json | 7 + 15 files changed, 675 insertions(+) create mode 100644 test/python/test_decoder_state_input_ids.py create mode 100644 test/test_models/create_decoder_input_ids_test_models.py create mode 100644 test/test_models/multimodal-decoder-no-input-ids/dummy_embedding.onnx create mode 100644 test/test_models/multimodal-decoder-no-input-ids/dummy_text.onnx create mode 100644 test/test_models/multimodal-decoder-no-input-ids/dummy_vision.onnx create mode 100644 test/test_models/multimodal-decoder-no-input-ids/genai_config.json create mode 100644 test/test_models/multimodal-decoder-no-input-ids/tokenizer.json create mode 100644 test/test_models/multimodal-decoder-no-input-ids/tokenizer_config.json create mode 100644 test/test_models/multimodal-decoder-with-input-ids/dummy_embedding.onnx create mode 100644 test/test_models/multimodal-decoder-with-input-ids/dummy_text.onnx create mode 100644 test/test_models/multimodal-decoder-with-input-ids/dummy_vision.onnx create mode 100644 test/test_models/multimodal-decoder-with-input-ids/genai_config.json create mode 100644 test/test_models/multimodal-decoder-with-input-ids/tokenizer.json create mode 100644 test/test_models/multimodal-decoder-with-input-ids/tokenizer_config.json diff --git a/.gitignore b/.gitignore index bf39331a7e..126e217f45 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,11 @@ examples/csharp/ModelChat/models !test/test_models/qwen35-hybrid-preprocessing/ !test/test_models/qwen35-hybrid-preprocessing/*.onnx !test/test_models/mistral3-vision-preprocessing/ +!test/test_models/multimodal-decoder-no-input-ids/ +!test/test_models/multimodal-decoder-no-input-ids/* +!test/test_models/multimodal-decoder-with-input-ids/ +!test/test_models/multimodal-decoder-with-input-ids/* +!test/test_models/create_decoder_input_ids_test_models.py .ipynb_checkpoints/ /src/java/.gradle diff --git a/test/python/test_decoder_state_input_ids.py b/test/python/test_decoder_state_input_ids.py new file mode 100644 index 0000000000..aa00c8331f --- /dev/null +++ b/test/python/test_decoder_state_input_ids.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License + +"""Tests for the DecoderState input_ids injection fix (PR #2148). + +Bug: DecoderState constructor checked *combined* session_info_ (decoder + vision + +embedding) for HasInput('input_ids'). The embedding session always declares +input_ids, so the check incorrectly injected input_ids into the decoder for models +like Mistral3 whose decoder has no input_ids input — causing an ORT error +"Invalid Feed Input Name: input_ids". + +Fix: use a decoder-only SessionInfo for the HasInput('input_ids') check. + +Two test model variants (in test/test_models/): + - multimodal-decoder-no-input-ids/ Mistral3-like: embedding has input_ids, + decoder does NOT. + - multimodal-decoder-with-input-ids/ Gemma4-like: both embedding and decoder + declare input_ids. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import numpy as np +import pytest + +import onnxruntime_genai as og + + +def _run_text_generation(model_path: str) -> None: + """Load the model and run one round of greedy text generation (text-only, no image). + + Appends a single seed token then generates up to max_length tokens. + Raises if the underlying ORT sessions receive an unexpected input feed. + """ + model = og.Model(model_path) + params = og.GeneratorParams(model) + params.set_search_options(do_sample=False, max_length=5) + + generator = og.Generator(model, params) + # Feed one seed token (token id=2, within vocab_size=10) + generator.append_tokens(np.array([[2]], dtype=np.int32)) + + while not generator.is_done(): + generator.generate_next_token() + + +@pytest.mark.parametrize("relative_model_path", [Path("multimodal-decoder-no-input-ids")]) +def test_decoder_no_input_ids_does_not_inject_input_ids(test_data_path, relative_model_path): + """Mistral3-like model: decoder declares no input_ids input. + + With the fix, DecoderState uses decoder-only SessionInfo and does NOT inject + input_ids into decoder feeds. Generation must succeed. + + Without the fix, DecoderState would use combined session_info_ (which includes + the embedding session that always has input_ids) and incorrectly inject input_ids + into the decoder, causing ORT to raise "Invalid Feed Input Name: input_ids". + """ + model_path = os.fspath(Path(test_data_path) / relative_model_path) + if not os.path.exists(model_path): + pytest.skip(f"Test model not found: {model_path}") + + # Should not raise — decoder receives only the inputs it declared + _run_text_generation(model_path) + + +@pytest.mark.parametrize("relative_model_path", [Path("multimodal-decoder-with-input-ids")]) +def test_decoder_with_input_ids_receives_input_ids(test_data_path, relative_model_path): + """Gemma4-like model: decoder declares input_ids as one of its inputs. + + With the fix, DecoderState uses decoder-only SessionInfo and correctly injects + input_ids into decoder feeds because the decoder session declares it. + Generation must succeed. + """ + model_path = os.fspath(Path(test_data_path) / relative_model_path) + if not os.path.exists(model_path): + pytest.skip(f"Test model not found: {model_path}") + + # Should not raise — decoder receives input_ids because it declared it + _run_text_generation(model_path) diff --git a/test/test_models/create_decoder_input_ids_test_models.py b/test/test_models/create_decoder_input_ids_test_models.py new file mode 100644 index 0000000000..72e6fd15fb --- /dev/null +++ b/test/test_models/create_decoder_input_ids_test_models.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +""" +Creates minimal dummy multi-modal test models for testing DecoderState input_ids injection. + +Two model variants are generated: + - multimodal-decoder-no-input-ids/ (Mistral3-like: decoder has NO input_ids input) + - multimodal-decoder-with-input-ids/ (Gemma4-like: decoder HAS input_ids input) + +These are used by test/python/test_decoder_state_input_ids.py. +""" + +import json +import os + +import numpy as np +import onnx +from onnx import TensorProto, helper, numpy_helper + +HIDDEN_SIZE = 64 +VOCAB_SIZE = 10 +NUM_KV_HEADS = 4 +HEAD_SIZE = 16 +NUM_LAYERS = 1 +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +# --------------------------------------------------------------------------- +# ONNX initializer helpers +# --------------------------------------------------------------------------- + +def _logits_weight() -> onnx.TensorProto: + """Zero weight matrix for logit projection: [HIDDEN_SIZE, VOCAB_SIZE].""" + return numpy_helper.from_array( + np.zeros((HIDDEN_SIZE, VOCAB_SIZE), dtype=np.float32), name="logits_weight" + ) + + +def _kv_pad_constant() -> onnx.TensorProto: + """Pad descriptor that appends exactly 1 zero element on axis-2 (sequence axis). + + For a 4-D KV tensor [B, H, P, D] the pad vector layout is: + [begin_B, begin_H, begin_P, begin_D, end_B, end_H, end_P, end_D] + = [0, 0, 0, 0, 0, 0, 1, 0] + + This makes present.key.shape[2] == past.key.shape[2] + 1 each step, + which satisfies ORT-GenAI's KV-cache shape validation. + """ + return numpy_helper.from_array( + np.array([0, 0, 0, 0, 0, 0, 1, 0], dtype=np.int64), name="kv_pad" + ) + + +# --------------------------------------------------------------------------- +# Shared decoder building block +# --------------------------------------------------------------------------- + +def _kv_inputs() -> list: + """KV cache inputs shared by both decoder variants.""" + return [ + helper.make_tensor_value_info( + f"past_key_values.{i}.{k}", + TensorProto.FLOAT, + ["batch", NUM_KV_HEADS, "past_seq", HEAD_SIZE], + ) + for i in range(NUM_LAYERS) + for k in ("key", "value") + ] + + +def _build_decoder_graph(extra_graph_inputs: list) -> tuple: + """Return (nodes, initializers, outputs) for a decoder model. + + The decoder: + - Projects inputs_embeds to logits via a zero-weight MatMul. + - Grows KV cache by padding one zero slice on the sequence axis. + + extra_graph_inputs additional graph inputs beyond inputs_embeds + KV cache. + """ + nodes = [ + helper.make_node("MatMul", ["inputs_embeds", "logits_weight"], ["logits"]), + ] + initializers = [_logits_weight(), _kv_pad_constant()] + outputs = [ + helper.make_tensor_value_info("logits", TensorProto.FLOAT, ["batch", "seq", VOCAB_SIZE]), + ] + + for i in range(NUM_LAYERS): + for k in ("key", "value"): + past = f"past_key_values.{i}.{k}" + present = f"present.{i}.{k}" + nodes.append(helper.make_node("Pad", [past, "kv_pad"], [present])) + outputs.append( + helper.make_tensor_value_info( + present, + TensorProto.FLOAT, + ["batch", NUM_KV_HEADS, "total_seq", HEAD_SIZE], + ) + ) + + return nodes, initializers, outputs + + +# --------------------------------------------------------------------------- +# Decoder model factories +# --------------------------------------------------------------------------- + +def make_decoder_no_input_ids() -> onnx.ModelProto: + """Decoder that does NOT declare input_ids (Mistral3-like). + + Inputs: inputs_embeds [batch, seq, HIDDEN_SIZE] + past_key_values.{i}.key/value + Outputs: logits [batch, seq, VOCAB_SIZE] + present.{i}.key/value (KV cache grown +1 via Pad) + """ + inputs_embeds = helper.make_tensor_value_info( + "inputs_embeds", TensorProto.FLOAT, ["batch", "seq", HIDDEN_SIZE] + ) + nodes, initializers, outputs = _build_decoder_graph([]) + graph = helper.make_graph( + nodes, "decoder", [inputs_embeds] + _kv_inputs(), outputs, initializers + ) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +def make_decoder_with_input_ids() -> onnx.ModelProto: + """Decoder that DOES declare input_ids (Gemma4-like). + + Inputs: input_ids [batch, seq] + inputs_embeds [batch, seq, HIDDEN_SIZE] + past_key_values.{i}.key/value + Outputs: logits [batch, seq, VOCAB_SIZE] + present.{i}.key/value (KV cache grown +1 via Pad) + """ + input_ids = helper.make_tensor_value_info( + "input_ids", TensorProto.INT32, ["batch", "seq"] + ) + inputs_embeds = helper.make_tensor_value_info( + "inputs_embeds", TensorProto.FLOAT, ["batch", "seq", HIDDEN_SIZE] + ) + nodes, initializers, outputs = _build_decoder_graph(["input_ids"]) + graph = helper.make_graph( + nodes, "decoder", [input_ids, inputs_embeds] + _kv_inputs(), outputs, initializers + ) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +# --------------------------------------------------------------------------- +# Embedding and vision model factories (shared by both variants) +# --------------------------------------------------------------------------- + +def make_embedding_model() -> onnx.ModelProto: + """Embedding model: converts input_ids → inputs_embeds. + + Inputs: input_ids [batch, seq] + image_features [num_tokens, HIDDEN_SIZE] (declared but unused in + computation; required so that + session_info_.GetInputDataType("image_features") succeeds + when MultiModalFeatures allocates an empty features tensor + for text-only generation) + Outputs: inputs_embeds [batch, seq, HIDDEN_SIZE] (fixed zero initializer) + """ + embeds_init = numpy_helper.from_array( + np.zeros((1, 1, HIDDEN_SIZE), dtype=np.float32), name="inputs_embeds" + ) + input_ids = helper.make_tensor_value_info("input_ids", TensorProto.INT32, ["batch", "seq"]) + image_features = helper.make_tensor_value_info( + "image_features", TensorProto.FLOAT, ["num_tokens", HIDDEN_SIZE] + ) + embeds_out = helper.make_tensor_value_info( + "inputs_embeds", TensorProto.FLOAT, ["batch", "seq", HIDDEN_SIZE] + ) + graph = helper.make_graph([], "embedding", [input_ids, image_features], [embeds_out], [embeds_init]) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +def make_vision_model() -> onnx.ModelProto: + """Minimal vision model (not exercised by the text-only test). + + Inputs: pixel_values [num_images, max_crops, 3, height, width] + image_sizes [num_images, 2] + Outputs: image_features [num_tokens, HIDDEN_SIZE] (fixed zero initializer) + """ + feat_init = numpy_helper.from_array( + np.zeros((1, HIDDEN_SIZE), dtype=np.float32), name="image_features" + ) + pixel_values = helper.make_tensor_value_info( + "pixel_values", TensorProto.FLOAT, ["num_images", "max_crops", 3, "height", "width"] + ) + image_sizes = helper.make_tensor_value_info("image_sizes", TensorProto.INT64, ["num_images", 2]) + feat_out = helper.make_tensor_value_info("image_features", TensorProto.FLOAT, ["num_tokens", HIDDEN_SIZE]) + graph = helper.make_graph([], "vision", [pixel_values, image_sizes], [feat_out], [feat_init]) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +# --------------------------------------------------------------------------- +# Config and tokenizer helpers +# --------------------------------------------------------------------------- + +def make_genai_config(decoder_filename: str) -> dict: + """genai_config.json for a phi3v-type multimodal model with tiny dimensions.""" + return { + "model": { + "type": "phi3v", + "bos_token_id": 1, + "eos_token_id": 1, + "pad_token_id": 0, + "vocab_size": VOCAB_SIZE, + "context_length": 64, + "decoder": { + "filename": decoder_filename, + "hidden_size": HIDDEN_SIZE, + "head_size": HEAD_SIZE, + "num_attention_heads": NUM_KV_HEADS, + "num_key_value_heads": NUM_KV_HEADS, + "num_hidden_layers": NUM_LAYERS, + "inputs": { + "inputs_embeds": "inputs_embeds", + "input_ids": "input_ids", + "attention_mask": "attention_mask", + "past_key_names": "past_key_values.%d.key", + "past_value_names": "past_key_values.%d.value", + }, + "outputs": { + "logits": "logits", + "present_key_names": "present.%d.key", + "present_value_names": "present.%d.value", + }, + "session_options": { + "provider_options": [], + }, + }, + "embedding": { + "filename": "dummy_embedding.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "image_features", + }, + "outputs": { + "inputs_embeds": "inputs_embeds", + }, + }, + "vision": { + "filename": "dummy_vision.onnx", + "inputs": { + "pixel_values": "pixel_values", + "image_sizes": "image_sizes", + }, + "outputs": { + "image_features": "image_features", + }, + }, + }, + "search": { + "do_sample": False, + "max_length": 10, + "num_beams": 1, + "temperature": 1.0, + "top_k": 1, + "top_p": 1.0, + "past_present_share_buffer": False, + }, + } + + +def make_tokenizer_json() -> dict: + """Minimal HuggingFace tokenizer JSON with a tiny vocabulary.""" + vocab = {str(i): i for i in range(VOCAB_SIZE)} + return { + "version": "1.0", + "truncation": None, + "padding": None, + "added_tokens": [ + {"id": 0, "content": "", "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": True}, + {"id": 1, "content": "", "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": True}, + ], + "normalizer": None, + "pre_tokenizer": None, + "post_processor": None, + "decoder": None, + "model": { + "type": "BPE", + "dropout": None, + "unk_token": "", + "continuing_subword_prefix": None, + "end_of_word_suffix": None, + "fuse_unk": False, + "byte_fallback": False, + "vocab": vocab, + "merges": [], + }, + } + + +def make_tokenizer_config() -> dict: + return { + "bos_token": "", + "eos_token": "", + "model_max_length": 64, + "tokenizer_class": "PreTrainedTokenizerFast", + "unk_token": "", + } + + +# --------------------------------------------------------------------------- +# Directory creation +# --------------------------------------------------------------------------- + +def create_model_dir(output_dir: str, decoder_model: onnx.ModelProto, decoder_filename: str) -> None: + os.makedirs(output_dir, exist_ok=True) + + for model_obj, filename in [ + (decoder_model, decoder_filename), + (make_embedding_model(), "dummy_embedding.onnx"), + (make_vision_model(), "dummy_vision.onnx"), + ]: + onnx.checker.check_model(model_obj) + onnx.save(model_obj, os.path.join(output_dir, filename)) + + with open(os.path.join(output_dir, "genai_config.json"), "w") as f: + json.dump(make_genai_config(decoder_filename), f, indent=4) + + with open(os.path.join(output_dir, "tokenizer.json"), "w") as f: + json.dump(make_tokenizer_json(), f, indent=4) + + with open(os.path.join(output_dir, "tokenizer_config.json"), "w") as f: + json.dump(make_tokenizer_config(), f, indent=4) + + print(f"Created: {output_dir}") + + +def main() -> None: + # Mistral3-like: decoder does NOT declare input_ids + create_model_dir( + os.path.join(SCRIPT_DIR, "multimodal-decoder-no-input-ids"), + make_decoder_no_input_ids(), + "dummy_text.onnx", + ) + + # Gemma4-like: decoder DOES declare input_ids + create_model_dir( + os.path.join(SCRIPT_DIR, "multimodal-decoder-with-input-ids"), + make_decoder_with_input_ids(), + "dummy_text.onnx", + ) + + +if __name__ == "__main__": + main() diff --git a/test/test_models/multimodal-decoder-no-input-ids/dummy_embedding.onnx b/test/test_models/multimodal-decoder-no-input-ids/dummy_embedding.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4190e117af14e849d047bb78fdf7e5468962b7bb GIT binary patch literal 433 zcmd|z3Q9|gDQn=;;l~UW`UiknL)L3?c>*Azls^U3(+PdDvt&ZRfb?u8-08>I)?* z4UNUR?s)eq5Bz+t9-`lU_Q3`{TBVubK6?&W$U%>jd6b{8ESfI_s^w{^Q!1ue3^#C+ zQZD=@d-FFm&KS2xc7A}m-1sRCQBT*5vy~`cTb(Bw!Wd3}LIwSe9c{7VLjj@|vRM+c z_3*+>e>W=lTOfHWWtNCVPgP6jvU4-xXhvREz~Am03<~Rm$czM?Cj#b_+S84p8v8D kf?KFJIS#hk;6fN(TOiJH!(dmJ}ot|q_ilt*sFnwVT6My4=$d9 z%!<^U__D;D($r!h3obJbMj<0E9wDy0(%g7(fE9Cb3vuQqR>UV4x?+aR{+x zq-LgPlyI>Nv6g41lw?Gyb8$n0w>Yy3XpjV#7zewM5Sl?u985`CTxjtCR?CP`Taurh Png_DS!HI>7K|l}yh>JDl literal 0 HcmV?d00001 diff --git a/test/test_models/multimodal-decoder-no-input-ids/genai_config.json b/test/test_models/multimodal-decoder-no-input-ids/genai_config.json new file mode 100644 index 0000000000..c43e6aa185 --- /dev/null +++ b/test/test_models/multimodal-decoder-no-input-ids/genai_config.json @@ -0,0 +1,62 @@ +{ + "model": { + "type": "phi3v", + "bos_token_id": 1, + "eos_token_id": 1, + "pad_token_id": 0, + "vocab_size": 10, + "context_length": 64, + "decoder": { + "filename": "dummy_text.onnx", + "hidden_size": 64, + "head_size": 16, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "num_hidden_layers": 1, + "inputs": { + "inputs_embeds": "inputs_embeds", + "input_ids": "input_ids", + "attention_mask": "attention_mask", + "past_key_names": "past_key_values.%d.key", + "past_value_names": "past_key_values.%d.value" + }, + "outputs": { + "logits": "logits", + "present_key_names": "present.%d.key", + "present_value_names": "present.%d.value" + }, + "session_options": { + "provider_options": [] + } + }, + "embedding": { + "filename": "dummy_embedding.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "image_features" + }, + "outputs": { + "inputs_embeds": "inputs_embeds" + } + }, + "vision": { + "filename": "dummy_vision.onnx", + "inputs": { + "pixel_values": "pixel_values", + "image_sizes": "image_sizes" + }, + "outputs": { + "image_features": "image_features" + } + } + }, + "search": { + "do_sample": false, + "max_length": 10, + "num_beams": 1, + "temperature": 1.0, + "top_k": 1, + "top_p": 1.0, + "past_present_share_buffer": false + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-no-input-ids/tokenizer.json b/test/test_models/multimodal-decoder-no-input-ids/tokenizer.json new file mode 100644 index 0000000000..4e3a129ece --- /dev/null +++ b/test/test_models/multimodal-decoder-no-input-ids/tokenizer.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": "", + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "vocab": { + "0": 0, + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + "7": 7, + "8": 8, + "9": 9 + }, + "merges": [] + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-no-input-ids/tokenizer_config.json b/test/test_models/multimodal-decoder-no-input-ids/tokenizer_config.json new file mode 100644 index 0000000000..4ebfa6586a --- /dev/null +++ b/test/test_models/multimodal-decoder-no-input-ids/tokenizer_config.json @@ -0,0 +1,7 @@ +{ + "bos_token": "", + "eos_token": "", + "model_max_length": 64, + "tokenizer_class": "PreTrainedTokenizerFast", + "unk_token": "" +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-with-input-ids/dummy_embedding.onnx b/test/test_models/multimodal-decoder-with-input-ids/dummy_embedding.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4190e117af14e849d047bb78fdf7e5468962b7bb GIT binary patch literal 433 zcmd|z3Q9|gC#WX|q>~^`+_%6zq1IPy~^Ih?E`&6VJVrbPZcsUF(ugy!kQws2(j{ zakMg-XOr8T-w$-#&@hg^ E05P$c(EtDd literal 0 HcmV?d00001 diff --git a/test/test_models/multimodal-decoder-with-input-ids/dummy_vision.onnx b/test/test_models/multimodal-decoder-with-input-ids/dummy_vision.onnx new file mode 100644 index 0000000000000000000000000000000000000000..bc83db5d3478c6da1c3c616c889b1aebc1d94f8a GIT binary patch literal 462 zcmdhk;6fN(TOiJH!(dmJ}ot|q_ilt*sFnwVT6My4=$d9 z%!<^U__D;D($r!h3obJbMj<0E9wDy0(%g7(fE9Cb3vuQqR>UV4x?+aR{+x zq-LgPlyI>Nv6g41lw?Gyb8$n0w>Yy3XpjV#7zewM5Sl?u985`CTxjtCR?CP`Taurh Png_DS!HI>7K|l}yh>JDl literal 0 HcmV?d00001 diff --git a/test/test_models/multimodal-decoder-with-input-ids/genai_config.json b/test/test_models/multimodal-decoder-with-input-ids/genai_config.json new file mode 100644 index 0000000000..c43e6aa185 --- /dev/null +++ b/test/test_models/multimodal-decoder-with-input-ids/genai_config.json @@ -0,0 +1,62 @@ +{ + "model": { + "type": "phi3v", + "bos_token_id": 1, + "eos_token_id": 1, + "pad_token_id": 0, + "vocab_size": 10, + "context_length": 64, + "decoder": { + "filename": "dummy_text.onnx", + "hidden_size": 64, + "head_size": 16, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "num_hidden_layers": 1, + "inputs": { + "inputs_embeds": "inputs_embeds", + "input_ids": "input_ids", + "attention_mask": "attention_mask", + "past_key_names": "past_key_values.%d.key", + "past_value_names": "past_key_values.%d.value" + }, + "outputs": { + "logits": "logits", + "present_key_names": "present.%d.key", + "present_value_names": "present.%d.value" + }, + "session_options": { + "provider_options": [] + } + }, + "embedding": { + "filename": "dummy_embedding.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "image_features" + }, + "outputs": { + "inputs_embeds": "inputs_embeds" + } + }, + "vision": { + "filename": "dummy_vision.onnx", + "inputs": { + "pixel_values": "pixel_values", + "image_sizes": "image_sizes" + }, + "outputs": { + "image_features": "image_features" + } + } + }, + "search": { + "do_sample": false, + "max_length": 10, + "num_beams": 1, + "temperature": 1.0, + "top_k": 1, + "top_p": 1.0, + "past_present_share_buffer": false + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-with-input-ids/tokenizer.json b/test/test_models/multimodal-decoder-with-input-ids/tokenizer.json new file mode 100644 index 0000000000..4e3a129ece --- /dev/null +++ b/test/test_models/multimodal-decoder-with-input-ids/tokenizer.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": "", + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "vocab": { + "0": 0, + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + "7": 7, + "8": 8, + "9": 9 + }, + "merges": [] + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-with-input-ids/tokenizer_config.json b/test/test_models/multimodal-decoder-with-input-ids/tokenizer_config.json new file mode 100644 index 0000000000..4ebfa6586a --- /dev/null +++ b/test/test_models/multimodal-decoder-with-input-ids/tokenizer_config.json @@ -0,0 +1,7 @@ +{ + "bos_token": "", + "eos_token": "", + "model_max_length": 64, + "tokenizer_class": "PreTrainedTokenizerFast", + "unk_token": "" +} \ No newline at end of file