Add gemma4 unit tests

test/python/test_gemma4_models.py

@@ -0,0 +1,265 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License
+
+"""
+Unit tests for Gemma4 multimodal model.
+Tests cover model loading, text-only processing, and image understanding.
+
+This file can be used in two ways:
+1. As a pytest module: pytest test_gemma4_models.py --test_models=/path/to/test_models
+2. As a standalone runner: python test_gemma4_models.py --cwd test/python --test_models test/test_models
+"""
+
+import argparse
+import logging
+import os
+import sys
+from pathlib import Path
+
+import numpy as np
+import onnxruntime_genai as og
+import pytest
+from _test_utils import run_subprocess
+
+logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG)
+log = logging.getLogger("gemma4-tests")
+
+GEMMA4_MODEL_NAME = "gemma4-vision-preprocessing"
+
+
+def _get_gemma4_model_path(test_data_path):
+    """Return the Gemma4 model path, skipping if it doesn't exist."""
+    model_path = os.path.join(test_data_path, GEMMA4_MODEL_NAME)
+    if not os.path.exists(model_path):
+        pytest.skip(f"Gemma4 test model not found at {model_path}")
+    return model_path
+
+
+def _get_onnx_path(test_data_path, filename):
+    """Return a path to a dummy ONNX file under the Gemma4 model dir, skipping if missing."""
+    path = os.path.join(test_data_path, GEMMA4_MODEL_NAME, filename)
+    if not os.path.exists(path):
+        pytest.skip(f"Gemma4 ONNX file not found at {path}")
+    return path
+
+
+def _load_model_and_processor(test_data_path):
+    """Load the Gemma4 model and create its multimodal processor."""
+    model_path = _get_gemma4_model_path(test_data_path)
+    model = og.Model(model_path)
+    return model, model.create_multimodal_processor()
+
+
+def _to_numpy(tensor):
+    """Convert an onnxruntime-genai tensor to a numpy array."""
+    if hasattr(tensor, "as_numpy"):
+        return tensor.as_numpy()
+    if hasattr(tensor, "numpy"):
+        return tensor.numpy()
+    return np.array(tensor)
+
+
+def test_gemma4_model_load(test_data_path):
+    """Test that the Gemma4 model loads successfully."""
+    model_path = _get_gemma4_model_path(test_data_path)
+    model = og.Model(model_path)
+    assert model is not None
+
+
+def test_gemma4_text_only(test_data_path):
+    """Test text-only processing (no images)."""
+    _, processor = _load_model_and_processor(test_data_path)
+
+    inputs = processor("What is the capital of France?", images=None)
+
+    assert inputs is not None
+    assert "input_ids" in inputs
+
+    ids = _to_numpy(inputs["input_ids"])
+    assert len(ids.shape) == 2, f"input_ids should be 2D, got shape {ids.shape}"
+    assert ids.shape[0] == 1, f"input_ids batch dim should be 1, got {ids.shape[0]}"
+
+    # Expected: BOS (2) + "What is the capital of France?"
+    expected_ids = [2, 3689, 563, 506, 5279, 529, 7001, 236881]
+    assert list(ids[0]) == expected_ids, f"input_ids mismatch: got {list(ids[0])}, expected {expected_ids}"
+
+
+@pytest.mark.parametrize("relative_image_path", [Path("images") / "australia.jpg"])
+def test_gemma4_vision_basic(test_data_path, relative_image_path):
+    """Test basic image processing with Gemma4."""
+    _, processor = _load_model_and_processor(test_data_path)
+
+    image_path = os.fspath(Path(test_data_path) / relative_image_path)
+    images = og.Images.open(image_path)
+
+    inputs = processor("<|image|>Describe this image", images=images)
+
+    assert inputs is not None
+    assert "pixel_values" in inputs
+    assert "input_ids" in inputs
+
+    ids = _to_numpy(inputs["input_ids"])
+    assert len(ids.shape) == 2, f"input_ids should be 2D, got shape {ids.shape}"
+    assert ids.shape[0] == 1, f"input_ids batch dim should be 1, got {ids.shape[0]}"
+    assert ids.shape[1] > 0, "input_ids should not be empty"
+
+
+@pytest.mark.parametrize("relative_image_path", [Path("images") / "landscape.jpg"])
+def test_gemma4_vision_load_from_bytes(test_data_path, relative_image_path):
+    """Test loading images from bytes for Gemma4."""
+    _, processor = _load_model_and_processor(test_data_path)
+
+    image_path = os.fspath(Path(test_data_path) / relative_image_path)
+    with open(image_path, "rb") as f:
+        images = og.Images.open_bytes(f.read())
+
+    inputs = processor("<|image|>What is shown in this image?", images=images)
+
+    assert inputs is not None
+    assert "pixel_values" in inputs
+
+
+@pytest.mark.parametrize(
+    "relative_image_paths",
+    [[Path("images") / "australia.jpg", Path("images") / "landscape.jpg"]],
+)
+def test_gemma4_vision_multiple_images(test_data_path, relative_image_paths):
+    """Test processing multiple images with Gemma4."""
+    _, processor = _load_model_and_processor(test_data_path)
+
+    image_paths = [os.fspath(Path(test_data_path) / p) for p in relative_image_paths]
+    images = og.Images.open(*image_paths)
+
+    inputs = processor("<|image|><|image|>Compare these images", images=images)
+
+    assert inputs is not None
+    assert "pixel_values" in inputs
+    assert "input_ids" in inputs
+
+    ids = _to_numpy(inputs["input_ids"])
+    assert len(ids.shape) == 2, f"input_ids should be 2D, got shape {ids.shape}"
+    assert ids.shape[0] == 1, f"input_ids batch dim should be 1, got {ids.shape[0]}"
+    assert ids.shape[1] > 0, "input_ids should not be empty"
+
+
+@pytest.mark.parametrize("relative_image_path", [Path("images") / "australia.jpg"])
+def test_gemma4_processor_creates_token_type_ids(test_data_path, relative_image_path):
+    """Test that Gemma4 processor creates token_type_ids for image prompts."""
+    _, processor = _load_model_and_processor(test_data_path)
+
+    image_path = os.fspath(Path(test_data_path) / relative_image_path)
+    images = og.Images.open(image_path)
+
+    inputs = processor("<|image|>Describe this image", images=images)
+
+    assert inputs is not None
+    assert "token_type_ids" in inputs
+
+
+def test_gemma4_vision_model_io(test_data_path):
+    """Validate the vision ONNX model has expected inputs and outputs."""
+    onnx = pytest.importorskip("onnx")
+
+    model = onnx.load(_get_onnx_path(test_data_path, "dummy_vision.onnx"))
+    input_names = {inp.name for inp in model.graph.input}
+    output_names = {out.name for out in model.graph.output}
+
+    assert "pixel_values" in input_names
+    assert "pixel_position_ids" in input_names
+    assert "image_features" in output_names
+
+    pv_input = next(i for i in model.graph.input if i.name == "pixel_values")
+    assert pv_input.type.tensor_type.elem_type == onnx.TensorProto.FLOAT, \
+        "pixel_values must be float32"
+
+    dim0 = pv_input.type.tensor_type.shape.dim[0]
+    assert dim0.dim_param != "", \
+        f"pixel_values dim-0 should be dynamic, got static dim_value={dim0.dim_value}"
+
+
+def test_gemma4_embedding_model_io(test_data_path):
+    """Validate the embedding ONNX model has expected inputs and outputs."""
+    onnx = pytest.importorskip("onnx")
+
+    model = onnx.load(_get_onnx_path(test_data_path, "dummy_embedding.onnx"))
+    input_names = {inp.name for inp in model.graph.input}
+    output_names = {out.name for out in model.graph.output}
+
+    assert "input_ids" in input_names
+    assert "image_features" in input_names
+    assert "token_type_ids" in input_names, "Gemma4 embedding requires token_type_ids"
+    assert "inputs_embeds" in output_names
+
+
+def test_gemma4_text_model_io(test_data_path):
+    """Validate the text/decoder ONNX model has expected inputs and outputs."""
+    onnx = pytest.importorskip("onnx")
+
+    model = onnx.load(_get_onnx_path(test_data_path, "dummy_text.onnx"))
+    input_names = {inp.name for inp in model.graph.input}
+    output_names = {out.name for out in model.graph.output}
+
+    assert "inputs_embeds" in input_names, "Decoder must accept inputs_embeds"
+    assert "attention_mask" in input_names
+    assert "position_ids" in input_names
+    assert "past_key_values.0.key" in input_names, "Decoder must have KV cache inputs"
+    assert "past_key_values.0.value" in input_names
+
+    assert "logits" in output_names
+    assert "present.0.key" in output_names, "Decoder must have KV cache outputs"
+    assert "present.0.value" in output_names
+
+
+# Standalone runner functionality
+def run_gemma4_vision_tests(
+    cwd: str | bytes | os.PathLike,
+    log: logging.Logger,
+    test_models: str | bytes | os.PathLike,
+):
+    """Run the vision model tests using pytest."""
+    log.debug("Running: Gemma4 Vision Model Tests")
+
+    command = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-sv",
+        "test_gemma4_models.py",
+        "--test_models",
+        test_models,
+    ]
+    run_subprocess(command, cwd=cwd, log=log).check_returncode()
+
+
+def parse_arguments():
+    """Parse command line arguments for standalone execution."""
+    parser = argparse.ArgumentParser(description="Test runner for Gemma4 vision models")
+    parser.add_argument(
+        "--cwd",
+        help="Path to the current working directory",
+        default=Path(__file__).parent.resolve().absolute(),
+    )
+    parser.add_argument(
+        "--test_models",
+        help="Path to the test_models directory",
+        default=Path(__file__).parent.parent.resolve().absolute() / "test_models",
+    )
+    return parser.parse_args()
+
+
+def main():
+    """Main entry point for standalone execution."""
+    args = parse_arguments()
+
+    log.info("Running Gemma4 vision model tests")
+    log.info(f"Test models path: {args.test_models}")
+    log.info(f"Working directory: {args.cwd}")
+
+    run_gemma4_vision_tests(os.path.abspath(args.cwd), log, os.path.abspath(args.test_models))
+
+    log.info("All tests completed successfully!")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

test/python/test_onnxruntime_genai.py

@@ -9,6 +9,8 @@
 
 import onnxruntime_genai as og
 from _test_utils import download_models, run_subprocess
+from test_gemma4_models import run_gemma4_vision_tests
+from test_qwen_fara_models import run_qwen_fara_vision_tests
 
 logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG)
 log = logging.getLogger("onnxruntime-genai-tests")
@@ -105,6 +107,11 @@ def main():
 
     # Run ONNX Runtime GenAI tests
     run_onnxruntime_genai_api_tests(os.path.abspath(args.cwd), log, os.path.abspath(args.test_models))
+
+    # Run vision model tests (tests auto-skip if models are not present)
+    run_gemma4_vision_tests(os.path.abspath(args.cwd), log, os.path.abspath(args.test_models))
+    run_qwen_fara_vision_tests(os.path.abspath(args.cwd), log, os.path.abspath(args.test_models))
+
     if args.e2e:
         run_onnxruntime_genai_e2e_tests(os.path.abspath(args.cwd), log, output_paths)
 

test/test_models/gemma4-vision-preprocessing/genai_config.json

@@ -0,0 +1,82 @@
+{
+    "model": {
+        "bos_token_id": 2,
+        "context_length": 32768,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            },
+            "filename": "dummy_text.onnx",
+            "head_size": 256,
+            "hidden_size": 2048,
+            "inputs": {
+                "inputs_embeds": "inputs_embeds",
+                "attention_mask": "attention_mask",
+                "position_ids": "position_ids",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 8,
+            "num_hidden_layers": 1,
+            "num_key_value_heads": 4
+        },
+        "eos_token_id": [
+            1,
+            106
+        ],
+        "pad_token_id": 0,
+        "type": "gemma4",
+        "vocab_size": 262144,
+        "embedding": {
+            "filename": "dummy_embedding.onnx",
+            "inputs": {
+                "input_ids": "input_ids",
+                "image_features": "image_features"
+            },
+            "outputs": {
+                "inputs_embeds": "inputs_embeds"
+            },
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            }
+        },
+        "vision": {
+            "filename": "dummy_vision.onnx",
+            "config_filename": "processor_config.json",
+            "inputs": {
+                "pixel_values": "pixel_values",
+                "pixel_position_ids": "pixel_position_ids"
+            },
+            "outputs": {
+                "image_features": "image_features"
+            },
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "provider_options": []
+            }
+        }
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 32768,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 1,
+        "top_p": 1.0
+    }
+}

test/test_models/gemma4-vision-preprocessing/processor_config.json

@@ -0,0 +1,27 @@
+{
+  "processor": {
+    "name": "gemma4_image_processor",
+    "transforms": [
+      {
+        "operation": {
+          "name": "decode_image",
+          "type": "DecodeImage",
+          "attrs": {
+            "color_space": "RGB"
+          }
+        }
+      },
+      {
+        "operation": {
+          "name": "gemma4_image_transform",
+          "type": "Gemma4ImageTransform",
+          "attrs": {
+            "patch_size": 16,
+            "max_soft_tokens": 280,
+            "pooling_kernel_size": 3
+          }
+        }
+      }
+    ]
+  }
+}