NVIDIA · luyiyun1021 · Sep 3, 2026 · Aug 11, 2026 · Aug 12, 2026 · Aug 12, 2026
@@ -117,7 +117,50 @@ When served via `trtllm-serve`, the following OpenAI-compatible endpoints are av
 
 The asynchronous `/v1/videos` job advances through `GET /v1/videos/{id}`: `queued` → `generating` (model inference) → `postprocessing` (encode the media and/or write the output file) → `completed`. The `generating` → `postprocessing` transition marks the end of inference; the video is downloadable via `/content` once `completed`.
 
-`response_format="path"` returns the generated file's server-side path (under `TRTLLM_MEDIA_STORAGE_PATH`) for co-located clients, enabled by default. Set `TRTLLM_DISALLOW_LOCAL_MEDIA_PATH=1` to reject such requests with HTTP 400. See the [serve examples](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/visual_gen/serve) for the full `response_format` reference.
+`response_format="path"` returns the generated file's server-side path (under `TRTLLM_MEDIA_STORAGE_PATH`) for co-located clients, enabled by default. Set `TRTLLM_DISALLOW_LOCAL_MEDIA_PATH=1` to reject such requests with HTTP 400; the same switch also rejects a reference sent with `format="path"`, since both ask the server to trust a local filesystem path. See the [serve examples](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/visual_gen/serve) for the full `response_format` reference.
+
+### Reference Inputs
+
+Conditioning references are supplied through the typed fields `image_reference`, `video_reference`, and `audio_reference`. Each field takes a single reference or a list. A reference is `MediaRef(content=..., format=...)`, and `format` is required.
+
+| `format` | Content | Notes |
+|---|---|---|
+| `path` | A local file readable by the coordinator process | Bare path or `file://` URI. |
+| `url` | An `http(s)` URL | Fetched on the coordinator through the SSRF-guarded loader. |
+| `base64` | Base64 text | A `data:` URI is also accepted. |
+| `bytes` | Raw `bytes` | Python API only. |
+
+Every pipeline declares the reference slots and roles it accepts through `ref_slot_specs`, and a request is validated against that declaration before generation begins. References are resolved to raw bytes on the coordinator, so a worker never needs a filesystem shared with the client.
+
+Most models take a single reference whose role is unambiguous:
+
+```python
+from tensorrt_llm import VisualGen
+from tensorrt_llm.visual_gen import MediaRef
+
+vg = VisualGen(model="Wan-AI/Wan2.2-TI2V-5B-Diffusers")
+params = vg.default_params
+params.image_reference = MediaRef(content="start.png", format="path")
+output = vg.generate(inputs="the scene comes alive with gentle motion", params=params)
+```
+
+Models that accept the same modality in more than one role need `role`. Wan 2.1 I2V takes a first frame and an optional last frame:
+
+```python
+from tensorrt_llm import VisualGen
+from tensorrt_llm.visual_gen import MediaRef
+
+vg = VisualGen(model="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers")
+params = vg.default_params
+params.image_reference = [
+    MediaRef(content="start.png", format="path", role="first_frame"),
+    MediaRef(content="end.png", format="path", role="last_frame"),
+]
+```
+
+FLUX.2 and Qwen-Image-Edit accept a list of reference images on `image_reference`.
+
+The same fields carry references over `trtllm-serve`; see [`examples/visual_gen/serve/`](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/visual_gen/serve) for request examples.
 
 ## Optimizations
 

@@ -29,6 +29,7 @@
 
 from tensorrt_llm import VisualGen, VisualGenArgs
 from tensorrt_llm._torch.visual_gen.models.cosmos3.transfer import TRANSFER_HINT_KEYS
+from tensorrt_llm.visual_gen import MediaRef
 
 _SCRIPT_DIR = Path(__file__).resolve().parent
 _ACTION_MODES = ("policy", "forward_dynamics", "inverse_dynamics")
@@ -336,7 +337,7 @@ def main():
         "--image_path",
         type=str,
         default=None,
-        help="Optional conditioning image path or URL for I2V/TI2V",
+        help="Optional conditioning image path for I2V/TI2V",
     )
     parser.add_argument(
         "--output_path",
@@ -477,7 +478,7 @@ def main():
     # Query per-model defaults (resolution, steps, guidance, seed, etc.).
     params = visual_gen.default_params
     if image_path is not None:
-        params.image = image_path
+        params.image_reference = [MediaRef(content=image_path, format="path")]
 
     negative_prompt = resolve_negative_prompt(
         negative_prompt=args.negative_prompt,
@@ -514,7 +515,7 @@ def main():
         with open(args.action_json, encoding="utf-8") as f:
             params.extra_params["action"] = json.load(f)
     if args.video_path is not None:
-        params.extra_params["video"] = Path(args.video_path).read_bytes()
+        params.video_reference = [MediaRef(content=args.video_path, format="path")]
     if args.extra_params:
         # Merged last: explicit JSON wins over flag-derived values.
         params.extra_params.update(args.extra_params)

@@ -26,6 +26,7 @@
 from pathlib import Path
 
 from tensorrt_llm import VisualGen, VisualGenArgs
+from tensorrt_llm.visual_gen import MediaRef
 
 
 def _output_paths(output_path: str, num_images: int) -> str | list[str]:
@@ -118,7 +119,9 @@ def main():
     # Start from per-model defaults and override only user-provided request fields.
     params = visual_gen.default_params
     params.num_images_per_prompt = args.num_images_per_prompt
-    params.image = args.image
+    params.image_reference = (
+        [MediaRef(content=path, format="path") for path in args.image] if args.image else None
+    )
     if args.image:
         # Let FLUX.2 derive omitted dimensions from the first processed reference.
         params.height = args.height

@@ -24,6 +24,7 @@
 import argparse
 
 from tensorrt_llm import VisualGen, VisualGenArgs
+from tensorrt_llm.visual_gen import MediaRef
 
 
 def parse_args() -> argparse.Namespace:
@@ -44,7 +45,7 @@ def parse_args() -> argparse.Namespace:
         "--image",
         nargs="+",
         required=True,
-        help="One or more input image paths or URLs.",
+        help="One or more input image paths.",
     )
     parser.add_argument(
         "--prompt",
@@ -64,7 +65,7 @@ def main() -> None:
     extra_args = VisualGenArgs.from_yaml(args.visual_gen_args) if args.visual_gen_args else None
     visual_gen = VisualGen(model=args.model, args=extra_args)
     params = visual_gen.default_params
-    params.image = args.image if len(args.image) > 1 else args.image[0]
+    params.image_reference = [MediaRef(content=path, format="path") for path in args.image]
     output = visual_gen.generate(inputs=args.prompt, params=params)
     saved = output.save(args.output_path)
     print(f"Saved edited image to {saved}")

@@ -24,6 +24,7 @@
 from pathlib import Path
 
 from tensorrt_llm import VisualGen, VisualGenArgs
+from tensorrt_llm.visual_gen import MediaRef
 
 
 def parse_args() -> argparse.Namespace:
@@ -63,7 +64,7 @@ def main() -> None:
     visual_gen = VisualGen(model=args.model, args=extra_args)
 
     params = visual_gen.default_params
-    params.image = args.image
+    params.image_reference = [MediaRef(content=args.image, format="path")]
 
     output = visual_gen.generate(inputs=args.prompt, params=params)
     if output.image is not None and output.image.shape[0] > 1:

@@ -24,6 +24,7 @@
 import os
 
 from tensorrt_llm import VisualGen, VisualGenArgs
+from tensorrt_llm.visual_gen import MediaRef
 
 _DEFAULT_IMAGE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "cat_piano.png")
 
@@ -62,9 +63,11 @@ def main():
     visual_gen = VisualGen(model=args.model, args=extra_args)
 
     # --- Model-specific: I2V request construction ---
-    # Start from per-model defaults (steps, guidance, seed, etc.) and set the input image.
+    # Start from per-model defaults (steps, guidance, seed, etc.) and set the
+    # first-frame reference. Wan I2V also accepts a ``last_frame`` role, so the
+    # role must be given to disambiguate.
     params = visual_gen.default_params
-    params.image = args.image
+    params.image_reference = [MediaRef(content=args.image, format="path", role="first_frame")]
 
     output = visual_gen.generate(
         inputs="A cat presses the piano keys with its paws, soft notes filling the quiet room.",

@@ -286,13 +286,31 @@ You can customize these by:
 - `frame_rate` (canonical) or `fps` (alias): frames per second
 - `num_frames`: when set, wins over the `seconds * frame_rate` derivation
 - `seed`, `num_inference_steps`, `guidance_scale`, `max_sequence_length`, `negative_prompt`: per-request denoise controls
-- `input_reference`: Reference image (I2V/TI2V) or video (V2V), accepted as a base64-encoded string in JSON or as a file in multipart form-data
+- `image_reference`, `video_reference`, `audio_reference`: reference image(s) for I2V/TI2V, video(s) for V2V, audio(s). In JSON each takes a `{content, format, role}` object or a list of them; a multipart file upload needs no `format`.
+  - `format` declares how to read `content`: `"path"` (a file readable by the server, or a `file://` URI), `"url"` (`http(s)`), or `"base64"` (or a `data:` URI). It is required in JSON, where `"bytes"` is rejected — upload the file instead.
+
+    ```json
+    {"image_reference": {"content": "iVBORw0KGgoAAAANSUhEUg...", "format": "base64"}}
+    ```
+
+  - `"path"` reads a file on the *server*, so it is only meaningful for a co-located client; set `TRTLLM_DISALLOW_LOCAL_MEDIA_PATH=1` to reject it (the same switch also disables `response_format="path"`). `"url"` is fetched through the SSRF-guarded loader (private-address block, redirect re-validation, timeout, size cap).
+  - `format` here is the *input* wire form; the top-level `format` selects the *output* encoding.
+  - `role` disambiguates a model that accepts the same modality in more than one role — Wan 2.1 I2V takes a first frame and an optional last frame. Roles and lists need a JSON body; a multipart upload is a single file with no role.
+
+    ```json
+    {"image_reference": [
+      {"content": "<base64>", "format": "base64", "role": "first_frame"},
+      {"content": "<base64>", "format": "base64", "role": "last_frame"}
+    ]}
+    ```
+
   - **Supported formats**: PNG and JPEG images; MP4 and AVI video, with H.264 the tested codec and others best-effort. HEIF/AVIF are not supported.
+- `input_reference` (deprecated): a single image or video reference, routed by content signature to I2V or V2V. A JSON request carries base64 bytes and a multipart request uploads the file; it is ignored when `image_reference` / `video_reference` is also given. Prefer the typed fields.
 - `extra_params`: model-specific overflow (see below)
 - `response_format`: `"file"` (default; `FileResponse` byte download) or `"path"` (server-side output path JSON, for co-located clients)
 - `format`: Generation content encoding. Video encoders: `"mp4"`, `"avi"`, `"auto"`. Tensor formats: `"safetensors"`, `"pt"` (carries video + audio + scalar metadata in one payload for LTX-2).
 
-> **`response_format="path"`** (image and video) returns absolute server-side file paths under the server's media-storage directory (`TRTLLM_MEDIA_STORAGE_PATH`), for clients co-located with the server (shared filesystem). Enabled by default; set `TRTLLM_DISALLOW_LOCAL_MEDIA_PATH=1` to reject `path` requests with HTTP 400.
+> **`response_format="path"`** (image and video) returns absolute server-side file paths under the server's media-storage directory (`TRTLLM_MEDIA_STORAGE_PATH`), for clients co-located with the server (shared filesystem). Enabled by default; set `TRTLLM_DISALLOW_LOCAL_MEDIA_PATH=1` to reject `path` requests with HTTP 400. One switch covers both directions: it also rejects a reference sent with `format="path"`.
 
 #### Tensor-format consumer contract
 
@@ -384,7 +402,7 @@ curl -X POST "http://localhost:8000/v1/videos" \
 ```bash
 curl -X POST "http://localhost:8000/v1/videos" \
   -F "prompt=She turns around and smiles" \
-  -F "input_reference=@./media/woman_skyline_original_720p.jpeg" \
+  -F "image_reference=@./media/woman_skyline_original_720p.jpeg" \
   -F "seconds=4.0" \
   -F "fps=24" \
   -F "size=256x256" \
@@ -393,11 +411,11 @@ curl -X POST "http://localhost:8000/v1/videos" \
 
 ### Video-to-Video (Multipart with File Upload, Cosmos3)
 ```bash
-# The reference is classified by content: image -> I2V, video -> V2V.
+# Modality comes from the field name: image_reference -> I2V, video_reference -> V2V.
 # V2V conditioning knobs ride in extra_params (values below are the defaults).
 curl -X POST "http://localhost:8000/v1/videos" \
   -F "prompt=Continue the same scene with smooth natural motion and consistent subjects." \
-  -F "input_reference=@./media/reference.mp4" \
+  -F "video_reference=@./media/reference.mp4" \
   -F "num_frames=189" \
   -F "fps=24" \
   -F 'extra_params={"condition_video_latent_indexes": [0, 1], "condition_video_keep": "first"}'

@@ -17,6 +17,8 @@
 """
 
 import argparse
+import base64
+import json
 import sys
 import time
 from pathlib import Path
@@ -28,7 +30,7 @@ def test_async_video_generation(
     base_url: str = "http://localhost:8000/v1",
     model: str = "wan",
     prompt: str = "A video of a cool cat on a motorcycle in the night",
-    input_reference: str = None,
+    image_reference: str = None,
     duration: float = 4.0,
     fps: int = 24,
     size: str = "256x256",
@@ -41,7 +43,7 @@ def test_async_video_generation(
         base_url: Base URL of the API server
         model: Model name to use
         prompt: Text prompt for generation
-        input_reference: Path to reference image (optional, for TI2V mode)
+        image_reference: Path to reference image (optional, for TI2V mode)
         duration: Video duration in seconds
         fps: Frames per second
         size: Video resolution (WxH format)
@@ -51,7 +53,7 @@ def test_async_video_generation(
         The server may return either MP4 (H.264) or AVI (MJPEG) format depending on
         the available encoder. The output filename extension will be adjusted to match.
     """
-    mode = "TI2V" if input_reference else "T2V"
+    mode = "TI2V" if image_reference else "T2V"
     print("=" * 80)
     print(f"Testing Async Video Generation API - {mode} Mode")
     print("=" * 80)
@@ -62,8 +64,8 @@ def test_async_video_generation(
     print("\n1. Creating video generation job...")
     print(f"   Mode: {mode}")
     print(f"   Prompt: {prompt}")
-    if input_reference:
-        print(f"   Input Reference: {input_reference}")
+    if image_reference:
+        print(f"   Input Reference: {image_reference}")
     print(f"   Duration: {duration}s")
     print(f"   FPS: {fps}")
     print(f"   Size: {size}")
@@ -81,14 +83,18 @@ def test_async_video_generation(
             },
         }
 
-        # Add input reference if provided (TI2V mode)
-        if input_reference:
-            if not Path(input_reference).exists():
-                print(f"\n❌ Error: Input reference image not found: {input_reference}")
+        # Add the conditioning image if provided (TI2V mode). The OpenAI SDK
+        # only knows one file parameter, `input_reference`, so the typed field
+        # travels base64-encoded in extra_body. It has to be a JSON string: a
+        # nested dict would be flattened into `image_reference[content]`.
+        if image_reference:
+            if not Path(image_reference).exists():
+                print(f"\n❌ Error: Input reference image not found: {image_reference}")
                 return False
-            create_params["input_reference"] = open(input_reference, "rb")
-
-        # Create video generation job
+            encoded = base64.b64encode(Path(image_reference).read_bytes()).decode()
+            create_params["extra_body"]["image_reference"] = json.dumps(
+                {"content": encoded, "format": "base64"}
+            )
         job = client.videos.create(**create_params)
 
         print("Video generation started: \n", job.model_dump_json(indent=2))
@@ -269,7 +275,7 @@ def test_async_video_generation(
         base_url=args.base_url,
         model=args.model,
         prompt=args.prompt,
-        input_reference=args.image,
+        image_reference=args.image,
         duration=args.duration,
         fps=args.fps,
         size=args.size,