[Bugfix] Fix MiniCPM-o 4.5 seeded turns and window E2E inputs (#8160)

Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>

tests/e2e/online_serving/helpers/minicpmo_window_e2e.py

@@ -38,10 +38,13 @@ async def run_window_turn(
         temperature=0.0,
         extra_body={
             "sliding_window_mode": mode,
-            "context_max_units": 1,
+            # Keep a complete ~5.5 s fixture utterance available to the
+            # native listen/speak policy. These limits still rebuild KV
+            # during the two-repeat input and its response.
+            "context_max_units": 8,
             "context_previous_max_tokens": 16,
-            "basic_window_high_tokens": 64,
-            "basic_window_low_tokens": 32,
+            "basic_window_high_tokens": 192,
+            "basic_window_low_tokens": 128,
         },
     )
     client = DuplexClient(
@@ -58,6 +61,10 @@ async def run_window_turn(
                 is_speech=True,
                 video_frames=video_frames,
             )
+            # The fixture ends in speech. A native duplex commit seals input
+            # but does not force an answer; keep the microphone clock running
+            # through a pause so the model can choose to speak.
+            await client.stream_pcm(bytes(5 * 32000), chunk_ms=200, realtime=True, is_speech=False)
             await client.commit(final=True)
             await wait_for_condition(
                 lambda: collector.count("input_audio_buffer.committed") > 0 or bool(collector.errors()),

tests/worker/test_native_duplex_hooks.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 import ast
 import inspect
+from dataclasses import replace
 from types import SimpleNamespace
 
 import numpy as np
@@ -1170,6 +1171,59 @@ def test_minicpmo_seeded_text_survives_explicit_initial_listen():
     assert state.pending_speech_context is True
 
 
+@pytest.mark.parametrize("initial_text", [None, "", "Please say hello.", "请说你好。"])
+@pytest.mark.parametrize("speech_envelope", [False, True])
+def test_minicpmo_seeded_context_keeps_assistant_turn_open_until_eos(initial_text, speech_envelope, mocker):
+    from vllm.v1.sample.metadata import SamplingMetadata
+
+    from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import _MiniCPMO45Stage0SessionState
+
+    runtime = _stage0_vision_runtime()
+    state = _MiniCPMO45Stage0SessionState(session_id="seeded-context")
+    runtime._prepare_session_context(state, {}, runtime_config={"initial_user_text": initial_text})
+    result = runtime._stage_prefill_embeddings_only(
+        state, np.zeros(4, dtype=np.float32), epoch=0, seq=1, is_speech=False
+    )
+    assert result["success"] is True
+    model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False})
+    logits = torch.zeros((1, 16), dtype=torch.float32)
+    logits[0, 10] = 20.0
+    original_logits = logits.clone()
+    sampling_metadata = mocker.Mock(spec=SamplingMetadata)
+    model.prepare_duplex_sampling(logits, sampling_metadata, (row,))
+    token_ids = model._minicpmo45_native_duplex_token_ids_cache
+    sampled = model._finalize_minicpmo45_native_duplex_sample(0, 7, token_ids)
+
+    if not initial_text:
+        assert sampled == 7
+        assert logits[0, 7].item() == 0.0
+        assert torch.isneginf(logits[0, :7]).all()
+        assert torch.isneginf(logits[0, 8:]).all()
+        return
+
+    assert torch.equal(logits, original_logits)
+    assert sampled == 8
+    if speech_envelope:
+        model._record_minicpmo45_duplex_terminator(0, 8, token_ids)
+        assert state.current_turn_ended is False
+        assert state.pending_speech_context is True
+    model._record_minicpmo45_duplex_terminator(0, 10, token_ids)
+    assert state.pending_speech_context is False
+    model._record_minicpmo45_duplex_terminator(0, 9, token_ids)
+
+    # A later silent unit must not re-admit the seed after the turn ended.
+    result = runtime._stage_prefill_embeddings_only(
+        state, np.zeros(4, dtype=np.float32), epoch=0, seq=2, is_speech=False
+    )
+    assert result["success"] is True
+    logits = original_logits.clone()
+    model.prepare_duplex_sampling(logits, sampling_metadata, (replace(row, seq=4),))
+    assert logits[0, 7].item() == 0.0
+    assert torch.isneginf(logits[0, :7]).all()
+    assert torch.isneginf(logits[0, 8:]).all()
+    assert model._finalize_minicpmo45_native_duplex_sample(0, 7, token_ids) == 7
+
+
 def test_minicpmo_stage0_puts_every_frame_of_an_append_in_one_unit():
     """Official streaming_prefill feeds the whole frame_list into one unit.
 

vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py

@@ -202,6 +202,11 @@ class MiniCPMO45Stage0DuplexRuntime:
             # Otherwise the turn-ended latch forces its silent input clock to
             # listen before the model can answer. Generated content clears it.
             state.pending_speech_context = True
+            # The seeded template already opens the assistant turn. Match
+            # that prefix in the native listen/speak state machine so silence
+            # advances this response until the model emits turn_eos.
+            state.current_turn_ended = False
+            state.pending_speech_response_open = True
 
     def _stage_prefill_embeddings_only(
         self,

vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py

@@ -989,9 +989,11 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
             with suppress(Exception):
                 state.pending_speech_response_open = False
             return
+        # A seeded prefix can open the response before tts_bos is sampled.
+        # Keep its pending input until the first content token in either case.
         if (
             sampled == tts_bos_id
-            and getattr(state, "current_turn_ended", True)
+            and (getattr(state, "current_turn_ended", True) or getattr(state, "pending_speech_response_open", False))
             and getattr(state, "pending_speech_context", False)
         ):
             with suppress(Exception):