diff --git a/.gitignore b/.gitignore index 87803c4a7b..d729fc4ad5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,11 @@ examples/csharp/ModelChat/models !test/test_models/qwen35-hybrid-preprocessing/ !test/test_models/qwen35-hybrid-preprocessing/*.onnx !test/test_models/mistral3-vision-preprocessing/ +!test/test_models/multimodal-decoder-no-input-ids/ +!test/test_models/multimodal-decoder-no-input-ids/* +!test/test_models/multimodal-decoder-with-input-ids/ +!test/test_models/multimodal-decoder-with-input-ids/* +!test/test_models/create_decoder_input_ids_test_models.py .ipynb_checkpoints/ /src/java/.gradle diff --git a/src/models/multi_modal.cpp b/src/models/multi_modal.cpp index 6a628168bd..541377646a 100644 --- a/src/models/multi_modal.cpp +++ b/src/models/multi_modal.cpp @@ -650,10 +650,17 @@ DecoderState::DecoderState(const MultiModalLanguageModel& model, DeviceSpanmodel.decoder.inputs.input_ids)) { - decoder_input_ids_ = std::make_unique(*this); - decoder_input_ids_->Add(); + // Some multimodal decoders (e.g., Gemma4) require input_ids alongside inputs_embeds. + // Use a decoder-only SessionInfo to avoid false positives: the combined session_info_ + // includes embedding session inputs (which always has input_ids), causing this check + // to incorrectly fire for models like mistral3 whose decoder has no input_ids input. + { + SessionInfo decoder_only_info; + decoder_only_info.Add(*model_.decoder_session_); + if (decoder_only_info.HasInput(model_.config_->model.decoder.inputs.input_ids)) { + decoder_input_ids_ = std::make_unique(*this); + decoder_input_ids_->Add(); + } } position_inputs_->Add(); diff --git a/test/python/test_decoder_state_input_ids.py b/test/python/test_decoder_state_input_ids.py new file mode 100644 index 0000000000..aa00c8331f --- /dev/null +++ b/test/python/test_decoder_state_input_ids.py @@ -0,0 +1,82 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License + +"""Tests for the DecoderState input_ids injection fix (PR #2148). + +Bug: DecoderState constructor checked *combined* session_info_ (decoder + vision + +embedding) for HasInput('input_ids'). The embedding session always declares +input_ids, so the check incorrectly injected input_ids into the decoder for models +like Mistral3 whose decoder has no input_ids input — causing an ORT error +"Invalid Feed Input Name: input_ids". + +Fix: use a decoder-only SessionInfo for the HasInput('input_ids') check. + +Two test model variants (in test/test_models/): + - multimodal-decoder-no-input-ids/ Mistral3-like: embedding has input_ids, + decoder does NOT. + - multimodal-decoder-with-input-ids/ Gemma4-like: both embedding and decoder + declare input_ids. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import numpy as np +import pytest + +import onnxruntime_genai as og + + +def _run_text_generation(model_path: str) -> None: + """Load the model and run one round of greedy text generation (text-only, no image). + + Appends a single seed token then generates up to max_length tokens. + Raises if the underlying ORT sessions receive an unexpected input feed. + """ + model = og.Model(model_path) + params = og.GeneratorParams(model) + params.set_search_options(do_sample=False, max_length=5) + + generator = og.Generator(model, params) + # Feed one seed token (token id=2, within vocab_size=10) + generator.append_tokens(np.array([[2]], dtype=np.int32)) + + while not generator.is_done(): + generator.generate_next_token() + + +@pytest.mark.parametrize("relative_model_path", [Path("multimodal-decoder-no-input-ids")]) +def test_decoder_no_input_ids_does_not_inject_input_ids(test_data_path, relative_model_path): + """Mistral3-like model: decoder declares no input_ids input. + + With the fix, DecoderState uses decoder-only SessionInfo and does NOT inject + input_ids into decoder feeds. Generation must succeed. + + Without the fix, DecoderState would use combined session_info_ (which includes + the embedding session that always has input_ids) and incorrectly inject input_ids + into the decoder, causing ORT to raise "Invalid Feed Input Name: input_ids". + """ + model_path = os.fspath(Path(test_data_path) / relative_model_path) + if not os.path.exists(model_path): + pytest.skip(f"Test model not found: {model_path}") + + # Should not raise — decoder receives only the inputs it declared + _run_text_generation(model_path) + + +@pytest.mark.parametrize("relative_model_path", [Path("multimodal-decoder-with-input-ids")]) +def test_decoder_with_input_ids_receives_input_ids(test_data_path, relative_model_path): + """Gemma4-like model: decoder declares input_ids as one of its inputs. + + With the fix, DecoderState uses decoder-only SessionInfo and correctly injects + input_ids into decoder feeds because the decoder session declares it. + Generation must succeed. + """ + model_path = os.fspath(Path(test_data_path) / relative_model_path) + if not os.path.exists(model_path): + pytest.skip(f"Test model not found: {model_path}") + + # Should not raise — decoder receives input_ids because it declared it + _run_text_generation(model_path) diff --git a/test/test_models/create_decoder_input_ids_test_models.py b/test/test_models/create_decoder_input_ids_test_models.py new file mode 100644 index 0000000000..72e6fd15fb --- /dev/null +++ b/test/test_models/create_decoder_input_ids_test_models.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +""" +Creates minimal dummy multi-modal test models for testing DecoderState input_ids injection. + +Two model variants are generated: + - multimodal-decoder-no-input-ids/ (Mistral3-like: decoder has NO input_ids input) + - multimodal-decoder-with-input-ids/ (Gemma4-like: decoder HAS input_ids input) + +These are used by test/python/test_decoder_state_input_ids.py. +""" + +import json +import os + +import numpy as np +import onnx +from onnx import TensorProto, helper, numpy_helper + +HIDDEN_SIZE = 64 +VOCAB_SIZE = 10 +NUM_KV_HEADS = 4 +HEAD_SIZE = 16 +NUM_LAYERS = 1 +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +# --------------------------------------------------------------------------- +# ONNX initializer helpers +# --------------------------------------------------------------------------- + +def _logits_weight() -> onnx.TensorProto: + """Zero weight matrix for logit projection: [HIDDEN_SIZE, VOCAB_SIZE].""" + return numpy_helper.from_array( + np.zeros((HIDDEN_SIZE, VOCAB_SIZE), dtype=np.float32), name="logits_weight" + ) + + +def _kv_pad_constant() -> onnx.TensorProto: + """Pad descriptor that appends exactly 1 zero element on axis-2 (sequence axis). + + For a 4-D KV tensor [B, H, P, D] the pad vector layout is: + [begin_B, begin_H, begin_P, begin_D, end_B, end_H, end_P, end_D] + = [0, 0, 0, 0, 0, 0, 1, 0] + + This makes present.key.shape[2] == past.key.shape[2] + 1 each step, + which satisfies ORT-GenAI's KV-cache shape validation. + """ + return numpy_helper.from_array( + np.array([0, 0, 0, 0, 0, 0, 1, 0], dtype=np.int64), name="kv_pad" + ) + + +# --------------------------------------------------------------------------- +# Shared decoder building block +# --------------------------------------------------------------------------- + +def _kv_inputs() -> list: + """KV cache inputs shared by both decoder variants.""" + return [ + helper.make_tensor_value_info( + f"past_key_values.{i}.{k}", + TensorProto.FLOAT, + ["batch", NUM_KV_HEADS, "past_seq", HEAD_SIZE], + ) + for i in range(NUM_LAYERS) + for k in ("key", "value") + ] + + +def _build_decoder_graph(extra_graph_inputs: list) -> tuple: + """Return (nodes, initializers, outputs) for a decoder model. + + The decoder: + - Projects inputs_embeds to logits via a zero-weight MatMul. + - Grows KV cache by padding one zero slice on the sequence axis. + + extra_graph_inputs additional graph inputs beyond inputs_embeds + KV cache. + """ + nodes = [ + helper.make_node("MatMul", ["inputs_embeds", "logits_weight"], ["logits"]), + ] + initializers = [_logits_weight(), _kv_pad_constant()] + outputs = [ + helper.make_tensor_value_info("logits", TensorProto.FLOAT, ["batch", "seq", VOCAB_SIZE]), + ] + + for i in range(NUM_LAYERS): + for k in ("key", "value"): + past = f"past_key_values.{i}.{k}" + present = f"present.{i}.{k}" + nodes.append(helper.make_node("Pad", [past, "kv_pad"], [present])) + outputs.append( + helper.make_tensor_value_info( + present, + TensorProto.FLOAT, + ["batch", NUM_KV_HEADS, "total_seq", HEAD_SIZE], + ) + ) + + return nodes, initializers, outputs + + +# --------------------------------------------------------------------------- +# Decoder model factories +# --------------------------------------------------------------------------- + +def make_decoder_no_input_ids() -> onnx.ModelProto: + """Decoder that does NOT declare input_ids (Mistral3-like). + + Inputs: inputs_embeds [batch, seq, HIDDEN_SIZE] + past_key_values.{i}.key/value + Outputs: logits [batch, seq, VOCAB_SIZE] + present.{i}.key/value (KV cache grown +1 via Pad) + """ + inputs_embeds = helper.make_tensor_value_info( + "inputs_embeds", TensorProto.FLOAT, ["batch", "seq", HIDDEN_SIZE] + ) + nodes, initializers, outputs = _build_decoder_graph([]) + graph = helper.make_graph( + nodes, "decoder", [inputs_embeds] + _kv_inputs(), outputs, initializers + ) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +def make_decoder_with_input_ids() -> onnx.ModelProto: + """Decoder that DOES declare input_ids (Gemma4-like). + + Inputs: input_ids [batch, seq] + inputs_embeds [batch, seq, HIDDEN_SIZE] + past_key_values.{i}.key/value + Outputs: logits [batch, seq, VOCAB_SIZE] + present.{i}.key/value (KV cache grown +1 via Pad) + """ + input_ids = helper.make_tensor_value_info( + "input_ids", TensorProto.INT32, ["batch", "seq"] + ) + inputs_embeds = helper.make_tensor_value_info( + "inputs_embeds", TensorProto.FLOAT, ["batch", "seq", HIDDEN_SIZE] + ) + nodes, initializers, outputs = _build_decoder_graph(["input_ids"]) + graph = helper.make_graph( + nodes, "decoder", [input_ids, inputs_embeds] + _kv_inputs(), outputs, initializers + ) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +# --------------------------------------------------------------------------- +# Embedding and vision model factories (shared by both variants) +# --------------------------------------------------------------------------- + +def make_embedding_model() -> onnx.ModelProto: + """Embedding model: converts input_ids → inputs_embeds. + + Inputs: input_ids [batch, seq] + image_features [num_tokens, HIDDEN_SIZE] (declared but unused in + computation; required so that + session_info_.GetInputDataType("image_features") succeeds + when MultiModalFeatures allocates an empty features tensor + for text-only generation) + Outputs: inputs_embeds [batch, seq, HIDDEN_SIZE] (fixed zero initializer) + """ + embeds_init = numpy_helper.from_array( + np.zeros((1, 1, HIDDEN_SIZE), dtype=np.float32), name="inputs_embeds" + ) + input_ids = helper.make_tensor_value_info("input_ids", TensorProto.INT32, ["batch", "seq"]) + image_features = helper.make_tensor_value_info( + "image_features", TensorProto.FLOAT, ["num_tokens", HIDDEN_SIZE] + ) + embeds_out = helper.make_tensor_value_info( + "inputs_embeds", TensorProto.FLOAT, ["batch", "seq", HIDDEN_SIZE] + ) + graph = helper.make_graph([], "embedding", [input_ids, image_features], [embeds_out], [embeds_init]) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +def make_vision_model() -> onnx.ModelProto: + """Minimal vision model (not exercised by the text-only test). + + Inputs: pixel_values [num_images, max_crops, 3, height, width] + image_sizes [num_images, 2] + Outputs: image_features [num_tokens, HIDDEN_SIZE] (fixed zero initializer) + """ + feat_init = numpy_helper.from_array( + np.zeros((1, HIDDEN_SIZE), dtype=np.float32), name="image_features" + ) + pixel_values = helper.make_tensor_value_info( + "pixel_values", TensorProto.FLOAT, ["num_images", "max_crops", 3, "height", "width"] + ) + image_sizes = helper.make_tensor_value_info("image_sizes", TensorProto.INT64, ["num_images", 2]) + feat_out = helper.make_tensor_value_info("image_features", TensorProto.FLOAT, ["num_tokens", HIDDEN_SIZE]) + graph = helper.make_graph([], "vision", [pixel_values, image_sizes], [feat_out], [feat_init]) + return helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)]) + + +# --------------------------------------------------------------------------- +# Config and tokenizer helpers +# --------------------------------------------------------------------------- + +def make_genai_config(decoder_filename: str) -> dict: + """genai_config.json for a phi3v-type multimodal model with tiny dimensions.""" + return { + "model": { + "type": "phi3v", + "bos_token_id": 1, + "eos_token_id": 1, + "pad_token_id": 0, + "vocab_size": VOCAB_SIZE, + "context_length": 64, + "decoder": { + "filename": decoder_filename, + "hidden_size": HIDDEN_SIZE, + "head_size": HEAD_SIZE, + "num_attention_heads": NUM_KV_HEADS, + "num_key_value_heads": NUM_KV_HEADS, + "num_hidden_layers": NUM_LAYERS, + "inputs": { + "inputs_embeds": "inputs_embeds", + "input_ids": "input_ids", + "attention_mask": "attention_mask", + "past_key_names": "past_key_values.%d.key", + "past_value_names": "past_key_values.%d.value", + }, + "outputs": { + "logits": "logits", + "present_key_names": "present.%d.key", + "present_value_names": "present.%d.value", + }, + "session_options": { + "provider_options": [], + }, + }, + "embedding": { + "filename": "dummy_embedding.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "image_features", + }, + "outputs": { + "inputs_embeds": "inputs_embeds", + }, + }, + "vision": { + "filename": "dummy_vision.onnx", + "inputs": { + "pixel_values": "pixel_values", + "image_sizes": "image_sizes", + }, + "outputs": { + "image_features": "image_features", + }, + }, + }, + "search": { + "do_sample": False, + "max_length": 10, + "num_beams": 1, + "temperature": 1.0, + "top_k": 1, + "top_p": 1.0, + "past_present_share_buffer": False, + }, + } + + +def make_tokenizer_json() -> dict: + """Minimal HuggingFace tokenizer JSON with a tiny vocabulary.""" + vocab = {str(i): i for i in range(VOCAB_SIZE)} + return { + "version": "1.0", + "truncation": None, + "padding": None, + "added_tokens": [ + {"id": 0, "content": "", "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": True}, + {"id": 1, "content": "", "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": True}, + ], + "normalizer": None, + "pre_tokenizer": None, + "post_processor": None, + "decoder": None, + "model": { + "type": "BPE", + "dropout": None, + "unk_token": "", + "continuing_subword_prefix": None, + "end_of_word_suffix": None, + "fuse_unk": False, + "byte_fallback": False, + "vocab": vocab, + "merges": [], + }, + } + + +def make_tokenizer_config() -> dict: + return { + "bos_token": "", + "eos_token": "", + "model_max_length": 64, + "tokenizer_class": "PreTrainedTokenizerFast", + "unk_token": "", + } + + +# --------------------------------------------------------------------------- +# Directory creation +# --------------------------------------------------------------------------- + +def create_model_dir(output_dir: str, decoder_model: onnx.ModelProto, decoder_filename: str) -> None: + os.makedirs(output_dir, exist_ok=True) + + for model_obj, filename in [ + (decoder_model, decoder_filename), + (make_embedding_model(), "dummy_embedding.onnx"), + (make_vision_model(), "dummy_vision.onnx"), + ]: + onnx.checker.check_model(model_obj) + onnx.save(model_obj, os.path.join(output_dir, filename)) + + with open(os.path.join(output_dir, "genai_config.json"), "w") as f: + json.dump(make_genai_config(decoder_filename), f, indent=4) + + with open(os.path.join(output_dir, "tokenizer.json"), "w") as f: + json.dump(make_tokenizer_json(), f, indent=4) + + with open(os.path.join(output_dir, "tokenizer_config.json"), "w") as f: + json.dump(make_tokenizer_config(), f, indent=4) + + print(f"Created: {output_dir}") + + +def main() -> None: + # Mistral3-like: decoder does NOT declare input_ids + create_model_dir( + os.path.join(SCRIPT_DIR, "multimodal-decoder-no-input-ids"), + make_decoder_no_input_ids(), + "dummy_text.onnx", + ) + + # Gemma4-like: decoder DOES declare input_ids + create_model_dir( + os.path.join(SCRIPT_DIR, "multimodal-decoder-with-input-ids"), + make_decoder_with_input_ids(), + "dummy_text.onnx", + ) + + +if __name__ == "__main__": + main() diff --git a/test/test_models/multimodal-decoder-no-input-ids/dummy_embedding.onnx b/test/test_models/multimodal-decoder-no-input-ids/dummy_embedding.onnx new file mode 100644 index 0000000000..4190e117af Binary files /dev/null and b/test/test_models/multimodal-decoder-no-input-ids/dummy_embedding.onnx differ diff --git a/test/test_models/multimodal-decoder-no-input-ids/dummy_text.onnx b/test/test_models/multimodal-decoder-no-input-ids/dummy_text.onnx new file mode 100644 index 0000000000..a1b27b1b1b Binary files /dev/null and b/test/test_models/multimodal-decoder-no-input-ids/dummy_text.onnx differ diff --git a/test/test_models/multimodal-decoder-no-input-ids/dummy_vision.onnx b/test/test_models/multimodal-decoder-no-input-ids/dummy_vision.onnx new file mode 100644 index 0000000000..bc83db5d34 Binary files /dev/null and b/test/test_models/multimodal-decoder-no-input-ids/dummy_vision.onnx differ diff --git a/test/test_models/multimodal-decoder-no-input-ids/genai_config.json b/test/test_models/multimodal-decoder-no-input-ids/genai_config.json new file mode 100644 index 0000000000..c43e6aa185 --- /dev/null +++ b/test/test_models/multimodal-decoder-no-input-ids/genai_config.json @@ -0,0 +1,62 @@ +{ + "model": { + "type": "phi3v", + "bos_token_id": 1, + "eos_token_id": 1, + "pad_token_id": 0, + "vocab_size": 10, + "context_length": 64, + "decoder": { + "filename": "dummy_text.onnx", + "hidden_size": 64, + "head_size": 16, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "num_hidden_layers": 1, + "inputs": { + "inputs_embeds": "inputs_embeds", + "input_ids": "input_ids", + "attention_mask": "attention_mask", + "past_key_names": "past_key_values.%d.key", + "past_value_names": "past_key_values.%d.value" + }, + "outputs": { + "logits": "logits", + "present_key_names": "present.%d.key", + "present_value_names": "present.%d.value" + }, + "session_options": { + "provider_options": [] + } + }, + "embedding": { + "filename": "dummy_embedding.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "image_features" + }, + "outputs": { + "inputs_embeds": "inputs_embeds" + } + }, + "vision": { + "filename": "dummy_vision.onnx", + "inputs": { + "pixel_values": "pixel_values", + "image_sizes": "image_sizes" + }, + "outputs": { + "image_features": "image_features" + } + } + }, + "search": { + "do_sample": false, + "max_length": 10, + "num_beams": 1, + "temperature": 1.0, + "top_k": 1, + "top_p": 1.0, + "past_present_share_buffer": false + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-no-input-ids/tokenizer.json b/test/test_models/multimodal-decoder-no-input-ids/tokenizer.json new file mode 100644 index 0000000000..4e3a129ece --- /dev/null +++ b/test/test_models/multimodal-decoder-no-input-ids/tokenizer.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": "", + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "vocab": { + "0": 0, + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + "7": 7, + "8": 8, + "9": 9 + }, + "merges": [] + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-no-input-ids/tokenizer_config.json b/test/test_models/multimodal-decoder-no-input-ids/tokenizer_config.json new file mode 100644 index 0000000000..4ebfa6586a --- /dev/null +++ b/test/test_models/multimodal-decoder-no-input-ids/tokenizer_config.json @@ -0,0 +1,7 @@ +{ + "bos_token": "", + "eos_token": "", + "model_max_length": 64, + "tokenizer_class": "PreTrainedTokenizerFast", + "unk_token": "" +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-with-input-ids/dummy_embedding.onnx b/test/test_models/multimodal-decoder-with-input-ids/dummy_embedding.onnx new file mode 100644 index 0000000000..4190e117af Binary files /dev/null and b/test/test_models/multimodal-decoder-with-input-ids/dummy_embedding.onnx differ diff --git a/test/test_models/multimodal-decoder-with-input-ids/dummy_text.onnx b/test/test_models/multimodal-decoder-with-input-ids/dummy_text.onnx new file mode 100644 index 0000000000..c92f14a036 Binary files /dev/null and b/test/test_models/multimodal-decoder-with-input-ids/dummy_text.onnx differ diff --git a/test/test_models/multimodal-decoder-with-input-ids/dummy_vision.onnx b/test/test_models/multimodal-decoder-with-input-ids/dummy_vision.onnx new file mode 100644 index 0000000000..bc83db5d34 Binary files /dev/null and b/test/test_models/multimodal-decoder-with-input-ids/dummy_vision.onnx differ diff --git a/test/test_models/multimodal-decoder-with-input-ids/genai_config.json b/test/test_models/multimodal-decoder-with-input-ids/genai_config.json new file mode 100644 index 0000000000..c43e6aa185 --- /dev/null +++ b/test/test_models/multimodal-decoder-with-input-ids/genai_config.json @@ -0,0 +1,62 @@ +{ + "model": { + "type": "phi3v", + "bos_token_id": 1, + "eos_token_id": 1, + "pad_token_id": 0, + "vocab_size": 10, + "context_length": 64, + "decoder": { + "filename": "dummy_text.onnx", + "hidden_size": 64, + "head_size": 16, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "num_hidden_layers": 1, + "inputs": { + "inputs_embeds": "inputs_embeds", + "input_ids": "input_ids", + "attention_mask": "attention_mask", + "past_key_names": "past_key_values.%d.key", + "past_value_names": "past_key_values.%d.value" + }, + "outputs": { + "logits": "logits", + "present_key_names": "present.%d.key", + "present_value_names": "present.%d.value" + }, + "session_options": { + "provider_options": [] + } + }, + "embedding": { + "filename": "dummy_embedding.onnx", + "inputs": { + "input_ids": "input_ids", + "image_features": "image_features" + }, + "outputs": { + "inputs_embeds": "inputs_embeds" + } + }, + "vision": { + "filename": "dummy_vision.onnx", + "inputs": { + "pixel_values": "pixel_values", + "image_sizes": "image_sizes" + }, + "outputs": { + "image_features": "image_features" + } + } + }, + "search": { + "do_sample": false, + "max_length": 10, + "num_beams": 1, + "temperature": 1.0, + "top_k": 1, + "top_p": 1.0, + "past_present_share_buffer": false + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-with-input-ids/tokenizer.json b/test/test_models/multimodal-decoder-with-input-ids/tokenizer.json new file mode 100644 index 0000000000..4e3a129ece --- /dev/null +++ b/test/test_models/multimodal-decoder-with-input-ids/tokenizer.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": "", + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "vocab": { + "0": 0, + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + "6": 6, + "7": 7, + "8": 8, + "9": 9 + }, + "merges": [] + } +} \ No newline at end of file diff --git a/test/test_models/multimodal-decoder-with-input-ids/tokenizer_config.json b/test/test_models/multimodal-decoder-with-input-ids/tokenizer_config.json new file mode 100644 index 0000000000..4ebfa6586a --- /dev/null +++ b/test/test_models/multimodal-decoder-with-input-ids/tokenizer_config.json @@ -0,0 +1,7 @@ +{ + "bos_token": "", + "eos_token": "", + "model_max_length": 64, + "tokenizer_class": "PreTrainedTokenizerFast", + "unk_token": "" +} \ No newline at end of file