diff --git a/tests/e2e/online_serving/helpers/minicpmo_window_e2e.py b/tests/e2e/online_serving/helpers/minicpmo_window_e2e.py index 553f0799219..4b4b7231298 100644 --- a/tests/e2e/online_serving/helpers/minicpmo_window_e2e.py +++ b/tests/e2e/online_serving/helpers/minicpmo_window_e2e.py @@ -38,10 +38,13 @@ async def run_window_turn( temperature=0.0, extra_body={ "sliding_window_mode": mode, - "context_max_units": 1, + # Keep a complete ~5.5 s fixture utterance available to the + # native listen/speak policy. These limits still rebuild KV + # during the two-repeat input and its response. + "context_max_units": 8, "context_previous_max_tokens": 16, - "basic_window_high_tokens": 64, - "basic_window_low_tokens": 32, + "basic_window_high_tokens": 192, + "basic_window_low_tokens": 128, }, ) client = DuplexClient( @@ -58,6 +61,10 @@ async def run_window_turn( is_speech=True, video_frames=video_frames, ) + # The fixture ends in speech. A native duplex commit seals input + # but does not force an answer; keep the microphone clock running + # through a pause so the model can choose to speak. + await client.stream_pcm(bytes(5 * 32000), chunk_ms=200, realtime=True, is_speech=False) await client.commit(final=True) await wait_for_condition( lambda: collector.count("input_audio_buffer.committed") > 0 or bool(collector.errors()), diff --git a/tests/worker/test_native_duplex_hooks.py b/tests/worker/test_native_duplex_hooks.py index 6bc078227bb..65fb52ad017 100644 --- a/tests/worker/test_native_duplex_hooks.py +++ b/tests/worker/test_native_duplex_hooks.py @@ -5,6 +5,7 @@ import ast import inspect +from dataclasses import replace from types import SimpleNamespace import numpy as np @@ -1170,6 +1171,59 @@ def test_minicpmo_seeded_text_survives_explicit_initial_listen(): assert state.pending_speech_context is True +@pytest.mark.parametrize("initial_text", [None, "", "Please say hello.", "请说你好。"]) +@pytest.mark.parametrize("speech_envelope", [False, True]) +def test_minicpmo_seeded_context_keeps_assistant_turn_open_until_eos(initial_text, speech_envelope, mocker): + from vllm.v1.sample.metadata import SamplingMetadata + + from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState + + runtime = _stage0_vision_runtime() + state = _MiniCPMO45Stage0SessionState(session_id="seeded-context") + runtime._prepare_session_context(state, {}, runtime_config={"initial_user_text": initial_text}) + result = runtime._stage_prefill_embeddings_only( + state, np.zeros(4, dtype=np.float32), epoch=0, seq=1, is_speech=False + ) + assert result["success"] is True + model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False}) + logits = torch.zeros((1, 16), dtype=torch.float32) + logits[0, 10] = 20.0 + original_logits = logits.clone() + sampling_metadata = mocker.Mock(spec=SamplingMetadata) + model.prepare_duplex_sampling(logits, sampling_metadata, (row,)) + token_ids = model._minicpmo45_native_duplex_token_ids_cache + sampled = model._finalize_minicpmo45_native_duplex_sample(0, 7, token_ids) + + if not initial_text: + assert sampled == 7 + assert logits[0, 7].item() == 0.0 + assert torch.isneginf(logits[0, :7]).all() + assert torch.isneginf(logits[0, 8:]).all() + return + + assert torch.equal(logits, original_logits) + assert sampled == 8 + if speech_envelope: + model._record_minicpmo45_duplex_terminator(0, 8, token_ids) + assert state.current_turn_ended is False + assert state.pending_speech_context is True + model._record_minicpmo45_duplex_terminator(0, 10, token_ids) + assert state.pending_speech_context is False + model._record_minicpmo45_duplex_terminator(0, 9, token_ids) + + # A later silent unit must not re-admit the seed after the turn ended. + result = runtime._stage_prefill_embeddings_only( + state, np.zeros(4, dtype=np.float32), epoch=0, seq=2, is_speech=False + ) + assert result["success"] is True + logits = original_logits.clone() + model.prepare_duplex_sampling(logits, sampling_metadata, (replace(row, seq=4),)) + assert logits[0, 7].item() == 0.0 + assert torch.isneginf(logits[0, :7]).all() + assert torch.isneginf(logits[0, 8:]).all() + assert model._finalize_minicpmo45_native_duplex_sample(0, 7, token_ids) == 7 + + def test_minicpmo_stage0_puts_every_frame_of_an_append_in_one_unit(): """Official streaming_prefill feeds the whole frame_list into one unit. diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py index 52213814863..79e4a810952 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py @@ -202,6 +202,11 @@ def _prepare_session_context( # Otherwise the turn-ended latch forces its silent input clock to # listen before the model can answer. Generated content clears it. state.pending_speech_context = True + # The seeded template already opens the assistant turn. Match + # that prefix in the native listen/speak state machine so silence + # advances this response until the model emits turn_eos. + state.current_turn_ended = False + state.pending_speech_response_open = True def _stage_prefill_embeddings_only( self, diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py index 69655209dfb..e3854937139 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py @@ -989,9 +989,11 @@ def _record_minicpmo45_duplex_terminator(self, row_idx: int, sampled: int, token with suppress(Exception): state.pending_speech_response_open = False return + # A seeded prefix can open the response before tts_bos is sampled. + # Keep its pending input until the first content token in either case. if ( sampled == tts_bos_id - and getattr(state, "current_turn_ended", True) + and (getattr(state, "current_turn_ended", True) or getattr(state, "pending_speech_response_open", False)) and getattr(state, "pending_speech_context", False) ): with suppress(Exception):