[Bugfix] Bound Omni-DuplexEval judge frame resolution

tests/benchmarks/duplex/test_omni_duplex_eval.py

@@ -449,7 +449,13 @@ def test_frame_sample_content_passes_frames(tmp_path, monkeypatch):
     response.with_name("response.meta.json").write_text(json.dumps({"clock": "media"}), encoding="utf-8")
     video = tmp_path / "video.mp4"
     video.write_bytes(b"video")
-    monkeypatch.setattr(eval_module, "extract_jpeg", lambda *args, **kwargs: b"\xff\xd8jpeg")
+    max_sides = []
+
+    def fake_extract_jpeg(*args, max_side=None, **kwargs):
+        max_sides.append(max_side)
+        return b"\xff\xd8jpeg"
+
+    monkeypatch.setattr(eval_module, "extract_jpeg", fake_extract_jpeg)
 
     class Judge:
         def temporal(self, *args, **kwargs):
@@ -476,6 +482,9 @@ def content(self, prompt, video, frames=None, *, mode="video_url"):
         judge_video_mode="frame-sample",
     )
     assert score["content"]["frame_count"] == 2
+    # Temporal and content frames are both bounded by the default max side.
+    assert max_sides and set(max_sides) == {eval_module.DEFAULT_JUDGE_FRAME_MAX_SIDE}
+    assert score["judge_frame_max_side"] == eval_module.DEFAULT_JUDGE_FRAME_MAX_SIDE
 
 
 def _write_rtd_response(tmp_path):
@@ -494,7 +503,13 @@ def test_content_prompt_falls_back_to_frame_sample_on_judge_context_overflow(tmp
     # the whole evaluation run.
     import requests
 
-    monkeypatch.setattr(eval_module, "extract_jpeg", lambda *args, **kwargs: b"\xff\xd8jpeg")
+    max_sides = []
+
+    def fake_extract_jpeg(*args, max_side=None, **kwargs):
+        max_sides.append(max_side)
+        return b"\xff\xd8jpeg"
+
+    monkeypatch.setattr(eval_module, "extract_jpeg", fake_extract_jpeg)
     response, video = _write_rtd_response(tmp_path)
 
     modes = []
@@ -533,6 +548,8 @@ def content(self, prompt, video, frames=None, *, mode="video_url"):
     assert score["content"]["judge_input_fallback"] == "frame-sample"
     assert score["content"]["frame_count"] == 2
     assert score["content"]["content_score"] == 3
+    # The fallback frames are bounded like every other judge frame.
+    assert max_sides and set(max_sides) == {eval_module.DEFAULT_JUDGE_FRAME_MAX_SIDE}
 
 
 def test_judge_context_length_error_detection():

tests/benchmarks/duplex/test_omni_duplex_eval_judge.py

@@ -207,6 +207,45 @@ def test_successful_retry_counts_valid_zero_and_refreshes_trace(sample_paths, ju
     assert trace[0]["raw_response"] == '{"success_score": 0}'
 
 
+def _evaluate_with_cap(sample_paths, sample_id, max_side):
+    sample, response, score = sample_paths
+    path = score.with_name(f"{sample_id}.json")
+    judge = judge_module.DuplexJudge("http://unused", "judge")
+    evaluator.evaluate_sample(replace(sample, id=sample_id), response, path, judge, judge_frame_max_side=max_side)
+    return path
+
+
+@pytest.mark.parametrize("max_side", [None, 768], ids=["source", "capped"])
+def test_summary_records_a_uniform_judge_frame_max_side(sample_paths, judge_http, max_side):
+    judge_http.return_value = _reply('{"success_score": 1}')
+    _evaluate_with_cap(sample_paths, "first", max_side)
+    path = _evaluate_with_cap(sample_paths, "second", max_side)
+    summary = evaluator.summarize_scores(path.parent)
+    assert summary["samples"] == 2
+    assert summary["judge_frame_max_side"] == max_side
+
+
+def test_summary_rejects_mixed_judge_frame_max_side(sample_paths, judge_http):
+    judge_http.return_value = _reply('{"success_score": 1}')
+    _evaluate_with_cap(sample_paths, "first", 1920)
+    path = _evaluate_with_cap(sample_paths, "second", 768)
+    with pytest.raises(ValueError, match=r"mixes judge_frame_max_side values \[1920, 768\]"):
+        evaluator.summarize_scores(path.parent)
+
+
+def test_summary_treats_a_score_without_the_field_as_source_resolution(sample_paths, judge_http):
+    judge_http.return_value = _reply('{"success_score": 1}')
+    path = _evaluate_with_cap(sample_paths, "first", None)
+    legacy = json.loads(path.read_text())
+    del legacy["judge_frame_max_side"]
+    path.with_name("legacy.json").write_text(json.dumps(legacy))
+    assert evaluator.summarize_scores(path.parent)["judge_frame_max_side"] is None
+
+    _evaluate_with_cap(sample_paths, "capped", 1920)
+    with pytest.raises(ValueError, match="mixes judge_frame_max_side"):
+        evaluator.summarize_scores(path.parent)
+
+
 @pytest.mark.parametrize(
     ("replies", "missing_field"),
     [
@@ -220,7 +259,7 @@ def test_rtd_requires_complete_scores_before_publishing(sample_paths, judge_http
     _, response, score = sample_paths
     sample = DuplexSample.from_row({"id": "sample", "split": "RTD_OCR", "video": "unused.mp4", "video_duration": 10})
     monkeypatch.setattr(evaluator, "materialize_media", lambda *args: "unused.mp4")
-    monkeypatch.setattr(evaluator, "_extract_frames", lambda *args: [b"jpeg"])
+    monkeypatch.setattr(evaluator, "_extract_frames", lambda *args, **kwargs: [b"jpeg"])
     judge_http.side_effect = [_reply(json.dumps(reply)) for reply in replies]
     if missing_field:
         with pytest.raises(ValueError, match=missing_field):

tests/benchmarks/duplex/test_omni_duplex_eval_media.py

@@ -80,7 +80,7 @@ def test_av_units_reject_invalid_pcm_or_unit_size(pcm, unit_ms):
 # ---------------------------------------------------------------------------
 
 
-def _make_synthetic_mp4(duration: float = 1.0, fps: int = 30) -> bytes:
+def _make_synthetic_mp4(duration: float = 1.0, fps: int = 30, width: int = 64, height: int = 64) -> bytes:
     """Return an MP4 byte blob with a solid-color pattern.
 
     Uses the built-in ``mpeg4`` encoder — available everywhere PyAV's bundled
@@ -90,11 +90,13 @@ def _make_synthetic_mp4(duration: float = 1.0, fps: int = 30) -> bytes:
     total_frames = int(duration * fps)
     with av.open(buf, mode="w", format="mp4") as container:
         stream = container.add_stream("mpeg4", rate=fps)
-        stream.width = 64
-        stream.height = 64
+        stream.width = width
+        stream.height = height
         stream.pix_fmt = "yuv420p"
         for i in range(total_frames):
-            frame = av.VideoFrame.from_ndarray(np.full((64, 64, 3), (i * 8) % 256, dtype=np.uint8), format="rgb24")
+            frame = av.VideoFrame.from_ndarray(
+                np.full((height, width, 3), (i * 8) % 256, dtype=np.uint8), format="rgb24"
+            )
             for pkt in stream.encode(frame):
                 container.mux(pkt)
         for pkt in stream.encode():
@@ -225,6 +227,24 @@ def test_negative_timestamp_clamped(tmp_path: Path) -> None:
         jpeg = extract_jpeg(path, timestamp=-1.0, quality=3)
         assert jpeg.startswith(b"\xff\xd8")
 
+    @staticmethod
+    def test_max_side_scales_down_keeping_aspect_ratio(tmp_path: Path) -> None:
+        from PIL import Image
+
+        path = tmp_path / "wide.mp4"
+        path.write_bytes(_make_synthetic_mp4(duration=0.5, fps=10, width=320, height=180))
+        full = Image.open(io.BytesIO(extract_jpeg(path, timestamp=0.0)))
+        scaled = Image.open(io.BytesIO(extract_jpeg(path, timestamp=0.0, max_side=160)))
+        assert full.size == (320, 180)
+        assert scaled.size == (160, 90)
+
+    @staticmethod
+    @pytest.mark.parametrize("max_side", [320, 640], ids=["equal", "larger"])
+    def test_max_side_leaves_frames_within_limit_unchanged(tmp_path: Path, max_side: int) -> None:
+        path = tmp_path / "small.mp4"
+        path.write_bytes(_make_synthetic_mp4(duration=0.5, fps=10, width=320, height=180))
+        assert extract_jpeg(path, timestamp=0.0, max_side=max_side) == extract_jpeg(path, timestamp=0.0)
+
     @staticmethod
     def test_raises_value_error_for_no_video(tmp_path: Path) -> None:
         wav = _make_synthetic_wav()

tests/e2e/features/fullduplex/test_omni_duplex_eval_cli.py

@@ -289,6 +289,42 @@ def unexpected_preparation(*args, **kwargs):
         )
 
 
+@pytest.mark.parametrize(
+    ("flag", "expected"),
+    [
+        ([], cli.DEFAULT_JUDGE_FRAME_MAX_SIDE),
+        (["--judge-frame-max-side", "768"], 768),
+        (["--judge-frame-max-side", "0"], None),
+    ],
+)
+def test_cli_evaluate_forwards_judge_frame_max_side(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, flag: list[str], expected: int | None
+):
+    manifest = tmp_path / "manifest.json"
+    manifest.write_text(json.dumps([{"id": "s", "split": "PR_correction", "question_text": "q"}]), encoding="utf-8")
+    seen = []
+    monkeypatch.setattr(cli, "DuplexJudge", lambda *args, **kwargs: object())
+    monkeypatch.setattr(cli, "evaluate_sample", lambda *args, **kwargs: seen.append(kwargs["judge_frame_max_side"]))
+
+    parser = argparse.ArgumentParser()
+    OmniDuplexEvalSubcommand.add_cli_args(parser)
+    args = ["evaluate", "--dataset", str(manifest), "--family", "pr", "--response-root", str(tmp_path / "r")]
+    args += ["--score-root", str(tmp_path / "s"), "--judge-model", "mock-judge", *flag]
+    OmniDuplexEvalSubcommand.cmd(parser.parse_args(args))
+    assert seen == [expected]
+
+
+def test_cli_evaluate_rejects_negative_judge_frame_max_side(tmp_path: Path):
+    manifest = tmp_path / "manifest.json"
+    manifest.write_text(json.dumps([{"id": "s", "split": "PR_correction", "question_text": "q"}]), encoding="utf-8")
+    parser = argparse.ArgumentParser()
+    OmniDuplexEvalSubcommand.add_cli_args(parser)
+    args = ["evaluate", "--dataset", str(manifest), "--family", "pr", "--response-root", str(tmp_path / "r")]
+    args += ["--score-root", str(tmp_path / "s"), "--judge-model", "mock-judge", "--judge-frame-max-side", "-1"]
+    with pytest.raises(ValueError, match="--judge-frame-max-side"):
+        OmniDuplexEvalSubcommand.cmd(parser.parse_args(args))
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("cached", [False, True])
 async def test_generate_rejects_unpaced_server_vad_before_preparation_or_cache(tmp_path, monkeypatch, cached):

vllm_omni/benchmarks/duplex/omni_duplex_eval_eval.py

@@ -50,6 +50,12 @@ def _write(path: Path, value: Any) -> None:
         temporary.unlink(missing_ok=True)
 
 
+# Judge frames are sent at source resolution unless bounded. Image tokens grow
+# with pixel count, so 4K frames overflow a 64K-token judge after ~10 frames
+# (#7375). 1920 keeps 1080p and smaller frames unchanged.
+DEFAULT_JUDGE_FRAME_MAX_SIDE = 1920
+
+
 def _text(items: list[dict[str, Any]]) -> str:
     return " ".join(str(item.get("sentence", item.get("text", ""))).strip() for item in items).strip()
 
@@ -80,6 +86,7 @@ def evaluate_sample(
     *,
     judge_fps: int = 2,
     judge_video_mode: str = "video_url",
+    judge_frame_max_side: int | None = DEFAULT_JUDGE_FRAME_MAX_SIDE,
     window_size: float = 10.0,
     allow_invalid_clock: bool = False,
 ) -> dict[str, Any]:
@@ -123,6 +130,7 @@ def judge_result(call: Callable[[], str], required: tuple[str, ...]) -> dict[str
         "response_meta": meta,
         "judge_model": getattr(judge, "model", None),
         "judge_video_mode": judge_video_mode,
+        "judge_frame_max_side": judge_frame_max_side,
     }
     if sample.family == "rtd":
         items = normalize_response_items(raw)
@@ -144,7 +152,7 @@ def judge_result(call: Callable[[], str], required: tuple[str, ...]) -> dict[str
                     }
                 )
                 continue
-            frames = _extract_frames(video_path, _times(*window, fps=judge_fps))
+            frames = _extract_frames(video_path, _times(*window, fps=judge_fps), max_side=judge_frame_max_side)
             parsed = judge_result(
                 lambda: judge.temporal(build_temporal_prompt(*window, item["sentence"], sample.question_text), frames),
                 ("temporal_score", "is_relevant"),
@@ -163,7 +171,7 @@ def judge_result(call: Callable[[], str], required: tuple[str, ...]) -> dict[str
             )
         content_frames = None
         if judge_video_mode == "frame-sample":
-            content_frames = _extract_frames(video_path, _content_frame_times(duration))
+            content_frames = _extract_frames(video_path, _content_frame_times(duration), max_side=judge_frame_max_side)
         content_prompt = build_content_prompt(
             _text(items), sample.question_text, _gt_references([sample.answer1, sample.answer2])
         )
@@ -185,7 +193,7 @@ def judge_result(call: Callable[[], str], required: tuple[str, ...]) -> dict[str
                 "retrying the content prompt in frame-sample mode",
                 sample.id,
             )
-            content_frames = _extract_frames(video_path, _content_frame_times(duration))
+            content_frames = _extract_frames(video_path, _content_frame_times(duration), max_side=judge_frame_max_side)
             if not content_frames:
                 # Nothing to fall back to: keep the original judge error.
                 raise
@@ -266,12 +274,12 @@ def _is_judge_context_length_error(exc: BaseException) -> bool:
     return "input length" in text and "exceed" in text
 
 
-def _extract_frames(path: str | Path, timestamps: list[float]) -> list[bytes]:
+def _extract_frames(path: str | Path, timestamps: list[float], *, max_side: int | None = None) -> list[bytes]:
     """Extract only decodable JPEG frames; clips can end on non-keyframes."""
     frames = []
     for timestamp in timestamps:
         try:
-            frame = extract_jpeg(path, timestamp=timestamp)
+            frame = extract_jpeg(path, timestamp=timestamp, max_side=max_side)
         except (OSError, ValueError):
             continue
         if frame.startswith(b"\xff\xd8"):
@@ -316,13 +324,24 @@ def summarize_scores(score_root: str | Path) -> dict[str, Any]:
             row = _read(path)
             row.setdefault("split", path.parent.name)
             rows.append(row)
+    # evaluate skips samples that already have a score, so a rerun with another
+    # cap would otherwise average different judge inputs. Scores from before the
+    # cap existed have no field and used source resolution, the same as None.
+    frame_max_sides = {row.get("judge_frame_max_side") for row in rows}
+    if len(frame_max_sides) > 1:
+        raise ValueError(
+            f"{score_root} mixes judge_frame_max_side values {sorted(frame_max_sides, key=str)}; "
+            "rerun evaluate with --overwrite and one value"
+        )
     rtd_temporal = [
         row["temporal"]["summary"] | {"content_score": row.get("content", {}).get("content_score", 0.0)}
         for row in rows
         if "temporal" in row
     ]
     pr = [{"task_type": row.get("task_type"), **row.get("pr", {})} for row in rows if "pr" in row]
     result = {"protocol_pin": PROTOCOL_PIN, "samples": len(rows)}
+    if rows:
+        result["judge_frame_max_side"] = frame_max_sides.pop()
     if rtd_temporal:
         by_task: dict[str, list[dict[str, float]]] = {}
         for row in rows:

vllm_omni/benchmarks/duplex/omni_duplex_eval_media.py

@@ -129,7 +129,7 @@ def video_duration(path: str | Path) -> float:
         raise
 
 
-def extract_jpeg(path: str | Path, *, timestamp: float, quality: int = 3) -> bytes:
+def extract_jpeg(path: str | Path, *, timestamp: float, quality: int = 3, max_side: int | None = None) -> bytes:
     """Extract a single JPEG frame at ``timestamp`` using PyAV.
 
     ``quality`` follows the ffmpeg ``-q:v`` convention (lower = better, 1..31)
@@ -141,6 +141,9 @@ def extract_jpeg(path: str | Path, *, timestamp: float, quality: int = 3) -> byt
     the ``ffmpeg -ss`` semantic of returning the first decodable frame *at or
     after* the requested timestamp.
 
+    If ``max_side`` is set, a frame whose longer side exceeds it is scaled down
+    to fit, keeping the aspect ratio. Smaller frames are left untouched.
+
     Raises:
         ValueError: If the file contains no video stream or no frame could be decoded.
     """
@@ -166,6 +169,8 @@ def extract_jpeg(path: str | Path, *, timestamp: float, quality: int = 3) -> byt
                 break
         if closest_frame is not None:
             image = closest_frame.to_image()
+            if max_side is not None and max(image.size) > max_side:
+                image.thumbnail((max_side, max_side))
             buf = io.BytesIO()
             image.save(buf, format="JPEG", quality=_ffmpeg_q_to_pil_quality(quality))
             return buf.getvalue()

vllm_omni/entrypoints/cli/benchmark/omni_duplex_eval.py

@@ -16,7 +16,11 @@
 from pathlib import Path
 
 from vllm_omni.benchmarks.duplex.omni_duplex_eval_dataset import DEFAULT_DATASET, DuplexSample, load_samples
-from vllm_omni.benchmarks.duplex.omni_duplex_eval_eval import evaluate_sample, summarize_scores
+from vllm_omni.benchmarks.duplex.omni_duplex_eval_eval import (
+    DEFAULT_JUDGE_FRAME_MAX_SIDE,
+    evaluate_sample,
+    summarize_scores,
+)
 from vllm_omni.benchmarks.duplex.omni_duplex_eval_judge import DuplexJudge
 from vllm_omni.benchmarks.duplex.omni_duplex_eval_runner import (
     DEFAULT_INSTRUCTIONS,
@@ -114,6 +118,12 @@ def add_cli_args(parser: argparse.ArgumentParser) -> None:
     evaluate.add_argument("--judge-api-key", default="EMPTY")
     evaluate.add_argument("--judge-video-mode", choices=("video_url", "frame-sample"), default="video_url")
     evaluate.add_argument("--judge-fps", type=int, default=2)
+    evaluate.add_argument(
+        "--judge-frame-max-side",
+        type=int,
+        default=DEFAULT_JUDGE_FRAME_MAX_SIDE,
+        help="Scale judge frames down so the longer side is at most this many pixels (0 keeps source resolution).",
+    )
     evaluate.add_argument("--window-size", type=float, default=10.0)
     evaluate.add_argument("--allow-invalid-clock", action="store_true")
     evaluate.add_argument("--eval-workers", type=int, default=1)
@@ -188,6 +198,8 @@ async def generate_one(sample: DuplexSample) -> GenerateSampleResult:
 
     if args.eval_workers < 1:
         raise ValueError("--eval-workers must be at least 1")
+    if args.judge_frame_max_side < 0:
+        raise ValueError("--judge-frame-max-side must be 0 or positive")
     judge = DuplexJudge(args.judge_base_url, args.judge_model, api_key=args.judge_api_key)
 
     def evaluate_one(sample: DuplexSample) -> None:
@@ -202,6 +214,7 @@ def evaluate_one(sample: DuplexSample) -> None:
             judge,
             judge_fps=args.judge_fps,
             judge_video_mode=args.judge_video_mode,
+            judge_frame_max_side=args.judge_frame_max_side or None,
             window_size=args.window_size,
             allow_invalid_clock=args.allow_invalid_clock,
         )