Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 136 additions & 13 deletions tests/benchmarks/duplex/test_omni_duplex_eval_drain_padding.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,34 @@
"""

import asyncio
import base64

import pytest

from vllm_omni.benchmarks.duplex import omni_duplex_eval_runner as runner
from vllm_omni.benchmarks.duplex.omni_duplex_eval_dataset import DuplexSample
from vllm_omni.benchmarks.duplex.omni_duplex_eval_runner import PreparedSample
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
from vllm_omni.experimental.fullduplex.client import RealtimeDuplexClient, RealtimeEventCollector

pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]


class _RecordingClient:
"""Records the commit and every appended unit, in order."""
"""Records the commit and every appended unit, in order.

Mirrors the real ``RealtimeDuplexClient`` surface: every coroutine defined
here exists on the real client too (see
``test_the_fake_client_mirrors_the_real_client_surface``), so these tests
cannot pass against a method the real client lacks.
"""

def __init__(self, *args, **kwargs) -> None:
self.events = RealtimeEventCollector()
self.calls: list[tuple[str, bool | None]] = []
self.calls: list[tuple[str, int | None]] = []
self.fail_appends = bool(kwargs.get("fail_appends"))
self.stall_commit = bool(kwargs.get("stall_commit"))
self.stall_stream = bool(kwargs.get("stall_stream"))
self.commit_gate = asyncio.Event()
self.events.add({"type": "session.created", "session": {"capabilities": {}}})
self.events.add({"type": "response.created", "response": {"id": "r1"}})
self.events.add({"type": "response.done", "response": {"id": "r1", "status": "completed"}})
Expand All @@ -44,18 +54,22 @@ async def configure(self, *args, **kwargs) -> None:
return None

async def stream_av_units(self, *args, **kwargs) -> int:
if self.stall_stream:
await asyncio.sleep(3600.0)
return 0

async def commit(self) -> None:
# A real commit is a socket write: it yields before it lands, so the
# padding would get its first unit in if it started too early.
if self.stall_commit:
await self.commit_gate.wait()
await asyncio.sleep(0.01)
self.calls.append(("commit", None))

async def append_audio(self, audio: bytes, *, is_speech: bool = True) -> None:
async def append_silence(self, duration_ms: int) -> None:
if self.fail_appends:
raise ConnectionResetError("socket closed")
self.calls.append(("append", is_speech))
self.calls.append(("append_silence", duration_ms))

async def acknowledge_playback(self) -> None:
return None
Expand All @@ -79,8 +93,8 @@ def _sample() -> DuplexSample:
)


def _prepared() -> PreparedSample:
return PreparedSample(pcm=b"", frames=(), duration=1.0, ref_audio=None, ref_audio_sha256=None)
def _prepared(duration: float = 1.0) -> PreparedSample:
return PreparedSample(pcm=b"", frames=(), duration=duration, ref_audio=None, ref_audio_sha256=None)


async def _instant_wait(check, *, timeout_s: float, label: str) -> bool:
Expand All @@ -92,13 +106,69 @@ async def test_padding_starts_only_after_the_commit() -> None:
client = _RecordingClient()
stop = asyncio.Event()

task = await runner._commit_clip_then_pad(client, 20, stop)
task, commit_timeout = await runner._commit_clip_then_pad(client, 20, stop)
await asyncio.sleep(0.1)
stop.set()
await asyncio.wait_for(task, timeout=1.0)

assert commit_timeout is None
assert client.calls[0] == ("commit", None)
assert client.calls[1:] and all(call == ("append", False) for call in client.calls[1:])
assert client.calls[1:] and all(call == ("append_silence", 20) for call in client.calls[1:])


def test_the_fake_client_mirrors_the_real_client_surface() -> None:
"""The fake must only define coroutines the real client also defines.

A fake ``append_audio`` once let these tests pass while the real
``RealtimeDuplexClient`` had no such method: the swallowed
``AttributeError`` ended the padding with zero units sent.
"""
real_coroutines = {name for name, attr in vars(RealtimeDuplexClient).items() if asyncio.iscoroutinefunction(attr)}
fake_coroutines = {name for name, attr in vars(_RecordingClient).items() if asyncio.iscoroutinefunction(attr)}

assert fake_coroutines <= real_coroutines


async def test_padding_works_against_the_real_client() -> None:
"""The padding must send real append events, not just fake-client calls.

Regression for the swallowed ``AttributeError``: only ``send`` is
stubbed, everything else — the event the padding builds included — is
the real client's code.
"""
client = RealtimeDuplexClient("ws://localhost:8000/v1/realtime")
sent: list[dict] = []

async def record_send(event: dict) -> None:
sent.append(event)

client.send = record_send
stop = asyncio.Event()
task = asyncio.create_task(runner._pad_silence_until_stopped(client, 20, stop))
try:
for _ in range(1000):
if sent:
break
await asyncio.sleep(0)
stop.set()
await asyncio.wait_for(task, timeout=1.0)
finally:
stop.set()
task.cancel()
try:
await task
except asyncio.CancelledError:
pass

assert sent, "the padding must send at least one unit on the real client"
first = sent[0]
assert first["type"] == "input_audio_buffer.append"
assert first["input_audio_format"] == "pcm16"
assert first["sample_rate_hz"] == runner.PCM16_SAMPLE_RATE
assert first["duration_ms"] == 20
assert first["audio_end_ms"] == 20
assert base64.b64decode(first["audio"]) == bytes(640) # 20 ms of zeroed PCM16
assert [event["audio_end_ms"] for event in sent] == sorted(event["audio_end_ms"] for event in sent)


async def test_sample_pads_only_after_its_commit(monkeypatch, tmp_path) -> None:
Expand All @@ -123,14 +193,16 @@ def client_factory(*args, **kwargs):

calls = created[0].calls
assert calls[0] == ("commit", None)
assert calls[1:] and all(call == ("append", False) for call in calls[1:])
# The padding period comes from the session's negotiated chunk period,
# not from the test: only its shape is pinned here.
assert calls[1:] and all(call[0] == "append_silence" and call[1] > 0 for call in calls[1:])


async def test_padding_stops_when_the_drain_finishes() -> None:
client = _RecordingClient()
stop = asyncio.Event()

task = await runner._commit_clip_then_pad(client, 20, stop)
task, _commit_timeout = await runner._commit_clip_then_pad(client, 20, stop)
await asyncio.sleep(0.1)
stop.set()
await asyncio.wait_for(task, timeout=1.0)
Expand All @@ -145,7 +217,7 @@ async def test_padding_sends_nothing_when_already_stopped() -> None:
stop = asyncio.Event()
stop.set()

task = await runner._commit_clip_then_pad(client, 20, stop)
task, _commit_timeout = await runner._commit_clip_then_pad(client, 20, stop)
await asyncio.wait_for(task, timeout=1.0)

assert client.calls == [("commit", None)]
Expand All @@ -155,8 +227,59 @@ async def test_padding_ends_quietly_when_the_socket_is_closed() -> None:
client = _RecordingClient(fail_appends=True)
stop = asyncio.Event()

task = await runner._commit_clip_then_pad(client, 20, stop)
task, _commit_timeout = await runner._commit_clip_then_pad(client, 20, stop)
# A closed socket ends the padding instead of failing the sample.
await asyncio.wait_for(task, timeout=1.0)

assert client.calls == [("commit", None)]


async def test_stalled_commit_is_bounded_and_padding_still_starts(monkeypatch) -> None:
"""A commit write that never lands costs a recorded timeout, not the sample."""
monkeypatch.setattr(runner, "_FINAL_COMMIT_TIMEOUT_S", 0.05)
client = _RecordingClient(stall_commit=True)
stop = asyncio.Event()

task, commit_timeout = await asyncio.wait_for(runner._commit_clip_then_pad(client, 20, stop), timeout=1.0)
await asyncio.sleep(0.05)
stop.set()
await asyncio.wait_for(task, timeout=1.0)
# Let the shielded commit write unwind so it leaves no pending task behind.
client.commit_gate.set()
await asyncio.sleep(0)

assert commit_timeout is not None
assert ("commit", None) not in client.calls # the stalled write never landed
assert client.calls and all(call == ("append_silence", 20) for call in client.calls)


async def test_stalled_input_is_bounded_and_the_sample_still_lands(monkeypatch, tmp_path) -> None:
"""A stalled input write is recorded; the sample still returns its artifacts."""
monkeypatch.setattr(runner, "_INPUT_SLACK_S", 0.05)
created: list[_RecordingClient] = []

def client_factory(*args, **kwargs):
client = _RecordingClient(*args, **{**kwargs, "stall_stream": True})
created.append(client)
return client

monkeypatch.setattr(runner, "RealtimeDuplexClient", client_factory)
monkeypatch.setattr(runner, "wait_for", _instant_wait)

result = await asyncio.wait_for(
runner.generate_sample(
_sample(),
url="ws://localhost:8000/v1/realtime",
model="test-model",
ref_audio=None,
output_root=tmp_path,
prepared=_prepared(duration=0.0),
),
timeout=5.0,
)

assert result.metadata["input_timeout"] is not None
assert result.metadata["input_committed"] is True # the commit itself was fine
assert result.metadata["response_done"] is True
assert result.metadata["input_timeout"] in result.error
assert result.output.exists() # the sample still lands its artifacts
19 changes: 19 additions & 0 deletions tests/benchmarks/duplex/test_omni_duplex_eval_judge_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,3 +119,22 @@ def test_immediate_score_json_skips_retry(tmp_path):
assert result["pr"]["all_success"] is True
assert len(judge.calls) == 1
assert len(_trace(tmp_path)) == 1


def test_parse_error_reaches_the_retry_and_the_trace(tmp_path):
"""The format retry carries the parse error itself, and the trace records it.

A judge reply can fail to parse without being prose — e.g. a malformed
value in a field that is present — so the retry has to say what failed,
and the first trace line has to show which reply failed to parse.
"""
judge = _FlakyJudge([_PROSE_REPLY, _SCORE_JSON])
result = evaluate_sample(_pr_sample(), _write_response(tmp_path), tmp_path / "512.json", judge)
assert result["pr"]["all_success"] is True

_, retried_followup = judge.calls[1]
assert "no score fields" in retried_followup[1]

first, _retried = _trace(tmp_path)
assert first["raw_response"] == _PROSE_REPLY
assert "no score fields" in first["error"]
Loading
Loading