[Frontend] Support image inputs in the Decisions API

benchmarks/decision_models/check_images.py

@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Exercise image content/order/repeat behavior across all decision types."""
+
+import argparse
+import json
+import math
+import struct
+import time
+import urllib.request
+import zlib
+from pathlib import Path
+
+import pybase64 as base64
+
+
+def png(width, height, color):
+    def chunk(kind, data):
+        return (
+            struct.pack(">I", len(data))
+            + kind
+            + data
+            + struct.pack(">I", zlib.crc32(kind + data))
+        )
+
+    raw = (b"\0" + bytes(color) * width) * height
+    data = b"\x89PNG\r\n\x1a\n" + chunk(
+        b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
+    )
+    return base64.b64encode(
+        data + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
+    ).decode()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--url", default="http://127.0.0.1:8000")
+    parser.add_argument("--model", required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    url = args.url.rstrip("/")
+    out = args.output
+    out.mkdir(parents=True, exist_ok=False)
+    questions = [
+        {"type": "predicate", "name": "red", "instructions": "Is the first image red?"},
+        {
+            "type": "choice",
+            "name": "color",
+            "instructions": "What color is the first image?",
+            "choices": [{"value": "red"}, {"value": "blue"}],
+        },
+        {
+            "type": "score",
+            "name": "brightness",
+            "instructions": "Rate image brightness.",
+            "levels": [{"label": "dark"}, {"label": "bright"}],
+        },
+    ]
+    results = []
+    for name, colors in [
+        ("red", [(255, 0, 0)]),
+        ("blue", [(0, 0, 255)]),
+        ("red-blue", [(255, 0, 0), (0, 0, 255)]),
+        ("blue-red", [(0, 0, 255), (255, 0, 0)]),
+        ("repeat-red", [(255, 0, 0)]),
+    ]:
+        payload = {
+            "model": args.model,
+            "input": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "input_text", "text": "Inspect the attached image(s)."}
+                    ]
+                    + [
+                        {
+                            "type": "input_image",
+                            "image_url": "data:image/png;base64," + png(128, 128, c),
+                        }
+                        for c in colors
+                    ],
+                }
+            ],
+            "questions": questions,
+        }
+        start = time.monotonic()
+        req = urllib.request.Request(
+            url + "/v1/decisions",
+            data=json.dumps(payload).encode(),
+            headers={"Content-Type": "application/json"},
+        )
+        with urllib.request.urlopen(req, timeout=600) as r:
+            response = json.load(r)
+        assert [a["name"] for a in response["answers"]] == [
+            "red",
+            "color",
+            "brightness",
+        ]
+        for a in response["answers"]:
+            probs = (
+                [a["probability"]]
+                if a["type"] == "predicate"
+                else [p["probability"] for p in a["probabilities"]]
+            )
+            assert all(math.isfinite(p) and 0 <= p <= 1 for p in probs)
+            if a["type"] != "predicate":
+                assert abs(sum(probs) - 1) < 1e-6
+        results.append(
+            {"case": name, "seconds": time.monotonic() - start, "response": response}
+        )
+        (out / "progress.json").write_text(json.dumps(results, indent=2))
+        print(name, "PASS", flush=True)
+    by_case = {r["case"]: r["response"]["answers"] for r in results}
+    for name in ("red", "red-blue"):
+        assert by_case[name][1]["choice"] == "red"
+    for name in ("blue", "blue-red"):
+        assert by_case[name][1]["choice"] == "blue"
+    for a, b in zip(by_case["red"], by_case["repeat-red"]):
+        values = (
+            lambda x: [x["probability"]]
+            if x["type"] == "predicate"
+            else [p["probability"] for p in x["probabilities"]]
+        )
+        assert max(abs(x - y) for x, y in zip(values(a), values(b))) < 1e-6
+    (out / "COMPLETE.json").write_text(
+        json.dumps(
+            {
+                "scope": (
+                    "Image API functional checks; "
+                    "not a quality or performance benchmark"
+                ),
+                "results": results,
+            },
+            indent=2,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

benchmarks/decision_models/vision-verification.json

@@ -0,0 +1,187 @@
+{
+  "scope": "GPU image API functional compatibility; synthetic color fixtures; no broad quality/calibration or resident performance claim",
+  "hardware": "RTX4070 12GB",
+  "runtime_base": "a4b14e57b6d5ad25a2215eff93d6f93ba05005df",
+  "vision_implementation": "90b0fa4",
+  "models": [
+    {
+      "family": "Gemma4",
+      "precision": "BF16 native safetensors",
+      "cpu_offload_gb_requested": 10,
+      "strategy": "Winnow canonical",
+      "source_manifest_sha256": "46ad81b27cc25d4f9e0c09de70ea2a4d457d0c37c6eb43e016b7e61c2cbea730",
+      "requests": 5,
+      "decisions": 15,
+      "repeated_probability_max_abs_delta": 2.2224553308802797e-09,
+      "functional_cases": [
+        "red",
+        "blue",
+        "red-blue",
+        "blue-red",
+        "repeat-red"
+      ],
+      "assertions_passed": [
+        "image content changes probabilities",
+        "swapping images changes first-image answer",
+        "repeating image preserves probabilities within 1e-6",
+        "predicate, choice and score responses validated"
+      ]
+    },
+    {
+      "family": "Qwen3.5",
+      "precision": "BF16 native safetensors",
+      "cpu_offload_gb_requested": 2,
+      "strategy": "generic next-token chat",
+      "source_manifest_sha256": "9b1460d98f84e1fcb24b9deccd48e77712ba867080400ec48d851c0fe6a82bff",
+      "requests": 5,
+      "decisions": 15,
+      "repeated_probability_max_abs_delta": 0.0,
+      "functional_cases": [
+        "red",
+        "blue",
+        "red-blue",
+        "blue-red",
+        "repeat-red"
+      ],
+      "assertions_passed": [
+        "image content changes probabilities",
+        "swapping images changes first-image answer",
+        "repeating image preserves probabilities within 1e-6",
+        "predicate, choice and score responses validated"
+      ]
+    }
+  ],
+  "cpu_processor_prefix_verification": [
+    {
+      "family": "gemma",
+      "actual_processor": "MULTIMODAL_REGISTRY native processor on CPU",
+      "scope": "Direct processor placeholder expansion, not full chat rendering or inference",
+      "two_images": true,
+      "same_images_private_questions_same_hashes": true,
+      "swapped_images_reverse_hashes": true,
+      "shared_token_prefix": 524,
+      "image_spans": [
+        {
+          "offset": 0,
+          "length": 258
+        },
+        {
+          "offset": 259,
+          "length": 258
+        }
+      ],
+      "image_prefix_end": 517
+    },
+    {
+      "family": "qwen",
+      "actual_processor": "MULTIMODAL_REGISTRY native processor on CPU",
+      "scope": "Direct processor placeholder expansion, not full chat rendering or inference",
+      "two_images": true,
+      "same_images_private_questions_same_hashes": true,
+      "swapped_images_reverse_hashes": true,
+      "shared_token_prefix": 140,
+      "image_spans": [
+        {
+          "offset": 1,
+          "length": 64
+        },
+        {
+          "offset": 68,
+          "length": 64
+        }
+      ],
+      "image_prefix_end": 132
+    }
+  ],
+  "native_online_renderer_prefix_verification": [
+    {
+      "family": "gemma",
+      "cases": 3,
+      "engine_keys": [
+        "type",
+        "prompt_token_ids",
+        "mm_kwargs",
+        "mm_hashes",
+        "mm_placeholders",
+        "prompt",
+        "arrival_time"
+      ],
+      "image_hash_order_and_repeat_verified": true,
+      "private_suffixes_verified": true,
+      "shared_token_prefix": 582,
+      "image_prefix_end": 570,
+      "image_spans": [
+        {
+          "offset": 53,
+          "length": 258
+        },
+        {
+          "offset": 312,
+          "length": 258
+        }
+      ]
+    },
+    {
+      "family": "qwen",
+      "cases": 3,
+      "engine_keys": [
+        "type",
+        "prompt_token_ids",
+        "mm_kwargs",
+        "mm_hashes",
+        "mm_placeholders",
+        "prompt",
+        "arrival_time"
+      ],
+      "image_hash_order_and_repeat_verified": true,
+      "private_suffixes_verified": true,
+      "shared_token_prefix": 141,
+      "image_prefix_end": 137,
+      "image_spans": [
+        {
+          "offset": 7,
+          "length": 64
+        },
+        {
+          "offset": 73,
+          "length": 64
+        }
+      ]
+    }
+  ],
+  "refreshed_stack_checks": [
+    {
+      "family": "Gemma4",
+      "implementation": "905e54a",
+      "requests": 5,
+      "decisions": 15,
+      "prefix_caching": true,
+      "repeat_max_abs_delta": 2.2224553308802797e-09,
+      "strict_repeat_tolerance": 1e-06,
+      "passed": true,
+      "artifact_sha256": "7a1c630ac460b3087c21f7a02ca75fbe9bf1cbf60aecf4eb29d30496c160335c"
+    },
+    {
+      "family": "Qwen3.5",
+      "implementation": "905e54a",
+      "requests": 5,
+      "decisions": 15,
+      "prefix_caching": false,
+      "repeat_max_abs_delta": 0.0,
+      "strict_repeat_tolerance": 1e-06,
+      "passed": true,
+      "artifact_sha256": "f56479005f4015e4b6b66a7bd8d902319605a0a0e308d8dfba13171c868e3bd4"
+    }
+  ],
+  "qwen_cached_repeat_limitation": {
+    "implementation": "905e54a",
+    "prefix_caching": true,
+    "requests_served": 5,
+    "color_order_checks_passed": true,
+    "repeat_max_abs_delta": 0.001278468154801926,
+    "strict_repeat_passed": false,
+    "batch_invariant_mode": "Startup rejected: VLLM batch_invariant mode is not supported for GDN_ATTN",
+    "native_renderer_token_hash_order_repeat_checks": "passed",
+    "mitigation_verified": "Disable prefix caching for strict repeatability on this tested configuration; no general determinism guarantee"
+  }
+}

docs/serving/online_serving/README.md

@@ -16,6 +16,8 @@ We currently support the following OpenAI APIs:
 - [Chat Completions batch API](./openai_compatible_server.md#chat-api) (`/v1/chat/completions/batch`)
 - [Responses API](./openai_compatible_server.md#responses-api) (`/v1/responses`, `/v1/responses/{response_id}`, `/v1/responses/{response_id}/cancel`)
     - Only applicable to [text generation models](../../models/generative_models.md).
+- [Decisions API](decisions.md) (`/v1/decisions`)
+    - Supports text-only predicate, choice, and score questions on supported generation models.
 - [Embeddings API](../../models/pooling_models/embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
     - Only applicable to [embedding models](../../models/pooling_models/embed.md).
 - [Transcriptions API](./speech_to_text.md#transcriptions-api) (`/v1/audio/transcriptions`)