Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions recipes/MiniMaxAI/MiniMax-H3.md
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,14 @@ curl -sS -X POST "${API_URL}" \
### 3. Ref2VA: image-only, image/audio, or mixed references

Run these requests against the combined service or a Ref2VA-only service.
`extra_params.ref_image_size` defaults to `match`, which scales each reference
image down to the output frame's pixel area while
preserving its aspect ratio. Set it to `max` to retain more reference detail,
up to a 2048-pixel short edge, at a higher compute cost. Neither mode enlarges
small images apart from 32-pixel alignment. For example, add
`"ref_image_size":"max"` to the request's `extra_params`; no server flag is needed.
This changes the previous fixed-2048 preprocessing and can change generated results.

Image-only Ref2VA omits `audio_reference`; adding one or more audio references
is optional. The typed fields accept one object or an ordered JSON list.
`audio_reference` accepts an HTTP(S) URL or a `data:` URL. In one terminal,
Expand Down Expand Up @@ -782,6 +790,8 @@ Run this request against the combined service. Repeat the
`input_references` multipart field once per source video. H3 consumes the
videos in form order and preserves their original soundtracks during
conditioning.
Reference videos use a 768-short-edge canvas capped at `768 * 1344` pixels;
smaller inputs keep their resolution apart from 32-pixel alignment.

```bash
export SUBJECT_VIDEO=/path/to/green_screen_subject.mp4
Expand Down
55 changes: 47 additions & 8 deletions tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def test_local_encoder_modes_preserve_validated_text_and_unified_handoff(task, t
(image.height // 32) * (image.width // 32), 96
)
pipeline.audio_vae = Mock()
prompt = {"prompt": "A person waves."}
prompt: dict[str, Any] = {"prompt": "A person waves."}
if task != "t2va":
prompt["multi_modal_data"] = {"image": Image.new("RGB", (256, 256))}
if text_key is not None:
Expand Down Expand Up @@ -1487,19 +1487,47 @@ def forward(self, query, key, value, metadata):
assert metadata.extra["valid_kv_length"] == 5


def test_reference_image_resize_contract():
@pytest.mark.parametrize(
("size", "target_pixels", "mode", "expected"),
[
((1344, 768), 1344 * 768, "match", (1344, 768)),
((640, 384), 1344 * 768, "match", (640, 384)),
((2688, 1536), 1344 * 768, "match", (1344, 768)),
((1536, 2688), 1344 * 768, "match", (768, 1344)),
((1344, 768), 448 * 256, "match", (448, 256)),
((1024, 1024), 1344 * 768, "match", (1024, 1024)),
((1344, 768), 448 * 256, "max", (1344, 768)),
((640, 384), 1344 * 768, "max", (640, 384)),
((3072, 3072), 1344 * 768, "max", (2048, 2048)),
((1080, 1440), 1344 * 768, "max", (1088, 1440)),
],
)
def test_reference_image_resize_contract(size, target_pixels, mode, expected):
from PIL import Image

from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
resolve_minimax_h3_reference_image_shape,
)

assert resolve_minimax_h3_reference_image_shape(Image.new("RGB", (1080, 1440))) == (
2048,
2720,
assert (
resolve_minimax_h3_reference_image_shape(
Image.new("RGB", size), target_pixels=target_pixels, ref_image_size=mode
)
== expected
)


def test_reference_image_resize_rejects_invalid_input():
from PIL import Image

from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
resolve_minimax_h3_reference_image_shape,
)

with pytest.raises(ValueError, match="aspect ratio"):
resolve_minimax_h3_reference_image_shape(Image.new("RGB", (100, 501)))
with pytest.raises(ValueError, match="dimensions"):
resolve_minimax_h3_reference_image_shape(Image.new("RGB", (128, 128)))


def test_fl2va_supports_first_last_and_explicit_frame_index_contracts():
Expand Down Expand Up @@ -1678,13 +1706,24 @@ def test_encoder_forward_forwards_video_inputs():
)


def test_reference_video_shape_uses_h3_adapt_shape_policy():
@pytest.mark.parametrize(
("size", "expected"),
[
((640, 384), (640, 384)),
((384, 640), (384, 640)),
((1280, 720), (1280, 704)),
((1344, 768), (1344, 768)),
((3844, 2160), (1344, 768)),
((2160, 3844), (768, 1344)),
((2048, 2048), (768, 768)),
],
)
def test_reference_video_shape_limits_resolution_without_upscaling(size, expected):
from vllm_omni.model_executor.models.minimax_h3.reference_video import (
_reference_video_shape,
)

assert _reference_video_shape(1280, 720) == (1344, 768)
assert _reference_video_shape(3844, 2160) == (1344, 768)
assert _reference_video_shape(*size) == expected


def test_text_encoder_stub_constructs_without_group_or_weights():
Expand Down
39 changes: 37 additions & 2 deletions tests/model_executor/stage_input_processors/test_minimax_h3.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""Regression tests for MiniMax H3's disaggregated encoder contract."""

from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock

import numpy as np
Expand Down Expand Up @@ -166,6 +167,40 @@ def test_prepare_ref2va_keeps_original_text_and_exact_condition_order():
assert len(media.audios) == 1


@pytest.mark.parametrize(("mode", "expected_size"), [(None, (448, 256)), ("match", (448, 256)), ("max", (1344, 768))])
def test_ref2va_reference_sizing_is_shared_by_qwen_and_vae(mode, expected_size):
from vllm_omni.model_executor.models.minimax_h3.encoder import MiniMaxH3Encoder
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import prepare_encoder_inputs

images = [Image.new("RGB", (1344, 768), color=color) for color in ("red", "green", "blue", "white")]
prompt = {"prompt": "Use all four references.", "multi_modal_data": {"image": images}}
extra_args = {"task": "ref2va"}
if mode is not None:
extra_args["ref_image_size"] = mode
sampling = OmniDiffusionSamplingParams(height=256, width=448, extra_args=extra_args)

local = prepare_encoder_inputs(prompt, sampling)
transformed = prepare_encoder_prompt(prompt, [sampling])
media = MiniMaxH3Encoder._media_input(transformed["additional_information"])
qwen_images = transformed["multi_modal_data"]["image"]
assert len(qwen_images) == len(media.images) == 4
assert (media.width, media.height) == (448, 256)
for index, qwen_image in enumerate(qwen_images):
assert qwen_image.size == expected_size
np.testing.assert_array_equal(np.asarray(qwen_image), media.images[index].numpy())
np.testing.assert_array_equal(np.asarray(qwen_image), np.asarray(local.images[index]))
torch.testing.assert_close(media.images[index], local.media.images[index])
assert [image.size for image in images] == [(1344, 768)] * 4


@pytest.mark.parametrize("mode", ["invalid", "MATCH", 2048, True, None, []])
def test_ref2va_rejects_invalid_reference_image_size(mode):
prompt = {"prompt": "hello", "multi_modal_data": {"image": Image.new("RGB", (256, 256))}}
sampling = OmniDiffusionSamplingParams(extra_args={"task": "ref2va", "ref_image_size": mode})
with pytest.raises(OmniClientError, match="ref_image_size must be match or max"):
prepare_encoder_prompt(prompt, [sampling])


def _mock_ref2va_video_with_audio(monkeypatch, *, duration_seconds: float) -> None:
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing

Expand Down Expand Up @@ -581,7 +616,7 @@ def test_full_payload_hook_skips_only_absent_output():
],
)
def test_encoder_handoff_rejects_invalid_full_payload(case, message, via_connector):
payload = _full_encoder_output()
payload: Any = _full_encoder_output()
layout = payload["kv_metadata"][MINIMAX_H3_ENCODER_LAYOUT_KEY]
if case == "not_mapping":
payload = []
Expand Down Expand Up @@ -621,7 +656,7 @@ def test_encoder_handoff_cleans_media_without_mutating_prompt_or_stripping_outpu
incoming = _full_encoder_output()
retained_hidden = torch.ones(1)
media = torch.zeros(2)
prompt = {
prompt: dict[str, Any] = {
"prompt": "hello",
"negative_prompt": "blur",
"multi_modal_data": {"image": object(), "audio": object(), "video": object()},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,13 +234,16 @@ def _prepare_encoder_images(
*,
height: int,
width: int,
ref_image_size: str = "match",
) -> list[Any]:
if not images:
return []
if task == "ref2va":
return [
image.resize(
resolve_minimax_h3_reference_image_shape(image),
resolve_minimax_h3_reference_image_shape(
image, target_pixels=height * width, ref_image_size=ref_image_size
),
Image.Resampling.LANCZOS,
)
for image in images
Expand Down Expand Up @@ -504,6 +507,7 @@ def prepare_encoder_inputs(
raw_images,
height=height,
width=width,
ref_image_size=extra_args.get("ref_image_size", "match"),
)
keyframe_indices = _resolve_fl2va_keyframe_indices(extra_args, len(images)) if task == "fl2va" else []
video_timestamps: list[list[float]] = []
Expand Down
21 changes: 18 additions & 3 deletions vllm_omni/model_executor/models/minimax_h3/preprocessing.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared MiniMax H3 media normalization and Qwen presentation building.

Builds the positive presentation token stream:
Expand Down Expand Up @@ -134,15 +135,29 @@ def resolve_minimax_h3_aspect_ratio(
return numeric_value


def resolve_minimax_h3_reference_image_shape(image: Image.Image) -> tuple[int, int]:
"""Resize an H3 reference image to the official 2048-short-edge canvas."""
def resolve_minimax_h3_reference_image_shape(
image: Image.Image,
*,
target_pixels: int = MINIMAX_H3_OUTPUT_MAX_PIXELS,
ref_image_size: str = "match",
) -> tuple[int, int]:
"""Limit reference resolution, then round each axis to the 32-pixel grid.

``match`` uses the generated frame's pixel area; ``max`` caps the short
edge at 2048. Neither mode upscales apart from alignment rounding.
"""
width, height = image.size
ratio = width / height
if not 0.4 <= ratio <= 2.5:
raise OmniClientError(f"reference image aspect ratio must be in [0.4, 2.5], got {width}x{height}")
if min(width, height) < 256 or max(width, height) > 5760:
raise OmniClientError(f"reference image dimensions must be in [256, 5760] pixels, got {width}x{height}")
scale = MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / min(width, height)
if ref_image_size == "match":
scale = min(1.0, math.sqrt(target_pixels / (width * height)))
elif ref_image_size == "max":
scale = min(1.0, MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / min(width, height))
else:
raise OmniClientError(f"MiniMax H3 ref_image_size must be match or max, got {ref_image_size!r}")
return (
_align_multiple(width * scale, MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE),
_align_multiple(height * scale, MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE),
Expand Down
11 changes: 10 additions & 1 deletion vllm_omni/model_executor/models/minimax_h3/reference_video.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""MiniMax H3 Ref2VA reference-video preparation."""

from __future__ import annotations
Expand Down Expand Up @@ -309,10 +310,18 @@ def _reference_video_shape(width: int, height: int) -> tuple[int, int]:
scale = math.sqrt(MINIMAX_H3_MAX_PIXELS / area)
target_width *= scale
target_height *= scale
return (
canvas_width, canvas_height = (
_nearest_multiple(target_width, MINIMAX_H3_CANVAS_MULTIPLE),
_nearest_multiple(target_height, MINIMAX_H3_CANVAS_MULTIPLE),
)
# Keep small reference videos at their own resolution, apart from
# rounding to the model's spatial grid.
if width * height < canvas_width * canvas_height:
return (
_nearest_multiple(width, MINIMAX_H3_CANVAS_MULTIPLE),
_nearest_multiple(height, MINIMAX_H3_CANVAS_MULTIPLE),
)
return canvas_width, canvas_height


def _transcode_reference_video(
Expand Down
Loading