[Bugfix][MiniCPM-o] Isolate duplex handoff failures and barge-in (#6170)

Signed-off-by: Sy03 <1370724210@qq.com>

tests/entrypoints/openai_api/test_duplex_handler.py

@@ -33,6 +33,7 @@ from vllm_omni.experimental.fullduplex.minicpmo45.runtime import (
 from vllm_omni.experimental.fullduplex.minicpmo45.session import (
     MiniCPMO45ServingSessionState,
 )
+from vllm_omni.experimental.fullduplex.openai import vad as realtime_vad
 from vllm_omni.experimental.fullduplex.openai.protocol import (
     DuplexCapabilities,
     DuplexOverlapPolicy,
@@ -706,7 +707,7 @@ async def test_native_realtime_protocol_preserves_input_turn_policy_hints():
 
 
 @pytest.mark.asyncio
-async def test_native_realtime_protocol_rejects_unimplemented_server_vad():
+async def test_native_realtime_protocol_rejects_server_vad_without_interrupt():
     ws = TimedWebSocket()
     protocol = NativeRealtimeSessionProtocol(ws)  # type: ignore[arg-type]
     protocol.bind_sender(ws.send_json)
@@ -729,7 +730,11 @@ async def test_native_realtime_protocol_rejects_unimplemented_server_vad():
     error = ws.sent[-1]
     assert error["type"] == "error"
     assert error["error"]["code"] == "unsupported_turn_detection"
-    assert "server_vad" in error["error"]["message"]
+    assert "interrupt_response=false" in error["error"]["message"]
+    threshold_error = protocol._validate_realtime_turn_detection(
+        {"turn_detection": {"type": "server_vad", "threshold": 0.15}}
+    )
+    assert threshold_error == "turn_detection.threshold must be greater than 0.15 and at most 1"
 
 
 @pytest.mark.asyncio
@@ -753,7 +758,7 @@ async def test_native_realtime_protocol_accepts_disabled_turn_detection():
 
 
 @pytest.mark.asyncio
-async def test_native_realtime_protocol_rejects_nested_vad_even_when_top_level_is_disabled():
+async def test_native_realtime_protocol_prefers_nested_vad_over_top_level():
     ws = TimedWebSocket()
     protocol = NativeRealtimeSessionProtocol(ws)  # type: ignore[arg-type]
     protocol.bind_sender(ws.send_json)
@@ -773,8 +778,55 @@ async def test_native_realtime_protocol_rejects_nested_vad_even_when_top_level_i
         }
     )
 
-    assert translated is None
-    assert ws.sent[-1]["error"]["code"] == "unsupported_turn_detection"
+    assert translated["session"]["extra_body"]["realtime_session_payload"]["overlap_policy"] == "barge_in_on_speech"
+
+
+@pytest.mark.asyncio
+async def test_realtime_vad_edge_order_and_tiny_chunk_resampling(monkeypatch):
+    ws = TimedWebSocket()
+    protocol = NativeRealtimeSessionProtocol(ws)  # type: ignore[arg-type]
+    protocol.bind_sender(ws.send_json)
+    protocol._server_vad = realtime_vad.SileroStreamingVAD(realtime_vad.SileroVADConfig())
+    result = realtime_vad.StreamingVADResult(True, True, True, True)
+    monkeypatch.setattr(protocol._server_vad, "process_base64", lambda *args, **kwargs: result)
+    protocol._input_speech_started = True
+    protocol._hold_realtime_output_until_session_created = False
+    await protocol._to_duplex_event(
+        {"type": "input_audio_buffer.append", "audio": _pcm_f32_b64(1), "format": "pcm_f32le"}
+    )
+    assert [event["type"] for event in ws.sent] == [
+        "input_audio_buffer.speech_stopped",
+        "input_audio_buffer.speech_started",
+    ]
+    ws.sent.clear()
+    protocol._input_speech_started = False
+    result = realtime_vad.StreamingVADResult(True, False, True, True)
+    await protocol._to_duplex_event({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"})
+    assert [event["type"] for event in ws.sent] == [
+        "input_audio_buffer.speech_started",
+        "input_audio_buffer.speech_stopped",
+    ]
+    detector = realtime_vad.SileroStreamingVAD(realtime_vad.SileroVADConfig(), frame_scorer=lambda _: 0.0)
+    for _ in range(1024):
+        detector.process_base64(_pcm_f32_b64(1), fmt="pcm_f32le", sample_rate_hz=48_000)
+    assert detector._processed_samples + detector._pending.size == 341
+
+
+@pytest.mark.asyncio
+async def test_realtime_server_vad_runtime_failures_are_terminal(monkeypatch):
+    handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
+    ws = TimedWebSocket()
+    ws.put(_server_vad_update("generic-vad", model="test-model", session_id="generic-vad"))
+    await handler.handle_session(ws, realtime_protocol=NativeRealtimeSessionProtocol({}))
+    assert ws.sent[-1]["error"]["code"] == "server_vad_requires_native_duplex"
+    monkeypatch.setattr(realtime_vad.SileroStreamingVAD, "process_base64", lambda *args, **kwargs: 1 / 0)
+    ws = TimedWebSocket()
+    create = _native_realtime_session_update("vad-inference-failure")
+    create["session"]["turn_detection"] = {"type": "server_vad"}
+    ws.put(create)
+    ws.put({"type": "input_audio_buffer.append", "audio": _pcm_f32_b64(1), "format": "pcm_f32le"})
+    await asyncio.wait_for(handler.handle_session(ws, realtime_protocol=NativeRealtimeSessionProtocol({})), timeout=2)
+    assert any(e.get("error", {}).get("code") == "realtime_input_failed" for e in ws.sent)
 
 
 def test_native_duplex_handler_has_no_fixed_session_admission_cap():
@@ -1551,6 +1603,14 @@ def _native_realtime_session_update(
     }
 
 
+def _server_vad_update(event_id: str, **session: object) -> dict[str, object]:
+    return {
+        "type": "session.update",
+        "event_id": event_id,
+        "session": {"turn_detection": {"type": "server_vad"}, **session},
+    }
+
+
 def test_native_stage0_request_id_matches_engine_after_same_session_id_reopen():
     handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
     first = handler._registry.create(session_id="sid-reused")
@@ -5311,7 +5371,7 @@ async def test_audio_clear_preserves_later_wire_order_append():
 
 
 @pytest.mark.asyncio
-async def test_native_append_is_cancelled_by_later_wire_order_input_cancel():
+async def test_native_append_is_cancelled_by_later_wire_order_input_cancel(mocker):
     class SlowAppendEngine(FakeEngineClient):
         def __init__(self) -> None:
             super().__init__()
@@ -5348,6 +5408,7 @@ async def test_native_append_is_cancelled_by_later_wire_order_input_cancel():
                 fence=fence,
             )
 
+    rollback = mocker.spy(MiniCPMO45PcmAppendBuffer, "_rollback_reservation")
     engine = SlowAppendEngine()
     handler = OmniDuplexSessionHandler(
         chat_service=FakeChatService(engine),
@@ -5378,6 +5439,7 @@ async def test_native_append_is_cancelled_by_later_wire_order_input_cancel():
     await asyncio.wait_for(handler_task, timeout=1)
 
     assert engine.append_cancelled.is_set()
+    rollback.assert_called_once()
     assert engine.signal_fences[-1] == DuplexFence("sid-cancel-slow-append")
     assert engine.signal_next_fences[-1] == DuplexFence("sid-cancel-slow-append", epoch=1)
 
@@ -5473,33 +5535,59 @@ async def test_native_session_update_commits_config_only_after_runtime_ack():
             return await super().signal_duplex_turn_async(session_id, **kwargs)
 
     engine = BlockingUpdateEngine()
-    handler = OmniDuplexSessionHandler(
-        chat_service=FakeChatService(engine),
-        config_timeout_s=0.1,
-        idle_timeout_s=1,
-    )
+    handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
     ws = TimedWebSocket()
-    ws.put(_session_create("sid-update-two-phase"))
-    ws.put(
-        {
-            "type": "turn.signal",
-            "event": "session.update",
-            "payload": {"instructions": "candidate instructions"},
-        }
-    )
+    protocol = NativeRealtimeSessionProtocol(ws)  # type: ignore[arg-type]
+    ws.put(_native_realtime_session_update("sid-update-two-phase"))
+    ws.put(_server_vad_update("update-two-phase"))
 
-    handler_task = asyncio.create_task(handler.handle_session(ws))
+    handler_task = asyncio.create_task(handler.handle_session(ws, realtime_protocol=protocol))
     await asyncio.wait_for(engine.update_started.wait(), timeout=1)
     live_session = handler._registry.get("sid-update-two-phase")
     assert live_session is not None
-    assert live_session.config.instructions != "candidate instructions"
+    assert live_session.config.overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
+    assert protocol._pending_turn_detection_update is not None
+    assert protocol._server_vad is None
+
+    protocol.encode_outbound_event({"type": "error", "code": "unrelated", "error": "unrelated"})
+    assert protocol._pending_turn_detection_update is not None
 
     engine.release_update.set()
     ws.put({"type": "session.close"})
     await asyncio.wait_for(handler_task, timeout=2)
 
-    updated = next(message for message in ws.sent if message.get("type") == "session.updated")
-    assert updated["session"]["instructions"] == "candidate instructions"
+    updated = next(message for message in reversed(ws.sent) if message.get("type") == "session.updated")
+    assert updated["session"]["overlap_policy"] == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value
+    assert isinstance(protocol._server_vad, realtime_vad.SileroStreamingVAD)
+
+
+@pytest.mark.parametrize(
+    ("append_fails", "expected_code"),
+    [(True, "session_update_aborted"), (False, "instructions_update_unsupported")],
+)
+@pytest.mark.asyncio
+async def test_realtime_vad_update_rejection_does_not_stall(append_fails, expected_code):
+    append_result = {"operation": "append", "ok": False, "error_count": 1} if append_fails else None
+    handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient(append_result=append_result)))
+    ws = TimedWebSocket()
+    protocol = NativeRealtimeSessionProtocol(ws)  # type: ignore[arg-type]
+    create = _native_realtime_session_update("sid-rejected-vad-update")
+    create["session"]["extra_body"]["auto_response"] = append_fails
+    ws.put(create)
+    if append_fails:
+        ws.put({"type": "input_audio_buffer.append", "audio": _pcm_f32_b64(16_000), "format": "pcm_f32le"})
+    update = _server_vad_update("rejected-vad-update")
+    if not append_fails:
+        update["session"]["instructions"] = "You are now a pirate."
+    ws.put(update)
+    ws.put({"type": "session.close"})
+    await asyncio.wait_for(handler.handle_session(ws, realtime_protocol=protocol), timeout=2)
+
+    errors = [message["error"] for message in ws.sent if isinstance(message.get("error"), dict)]
+    error = next(error for error in errors if error.get("code") == expected_code)
+    assert error["event_id"] == "rejected-vad-update"
+    assert protocol._pending_turn_detection_update is None
+    assert protocol._server_vad is None
 
 
 @pytest.mark.asyncio
@@ -5999,7 +6087,7 @@ async def test_minicpmo_native_duplex_rejects_ref_audio_path():
     assert engine.opened == []
 
 
-def test_minicpmo_native_duplex_explicit_barge_in_request_is_deferred_as_listen():
+def test_minicpmo_native_duplex_explicit_barge_in_request_interrupts():
     engine = FakeEngineClient()
     handler = OmniDuplexSessionHandler(
         chat_service=FakeChatService(engine),
@@ -6026,12 +6114,10 @@ def test_minicpmo_native_duplex_explicit_barge_in_request_is_deferred_as_listen(
     )
 
     assert decision == {
-        "action": "listen",
-        "reason": "barge_in_unsupported",
+        "action": "barge_in",
+        "reason": "client_force_barge_in",
         "duration_ms": 1000,
-        "overlap_speech_ms": 1000,
         "buffer_audio": True,
-        "defer_runtime_append": True,
     }
 
 
@@ -6043,7 +6129,7 @@ def test_minicpmo_native_duplex_explicit_barge_in_request_is_deferred_as_listen(
         {"type": "turn.signal", "event": "barge_in"},
     ],
 )
-async def test_minicpmo_native_duplex_rejects_unadvertised_barge_in_control(
+async def test_minicpmo_native_duplex_accepts_explicit_barge_in_control(
     barge_in_event: dict[str, object],
 ):
     engine = FakeEngineClient()
@@ -6059,9 +6145,8 @@ async def test_minicpmo_native_duplex_rejects_unadvertised_barge_in_control(
 
     await handler.handle_session(ws)
 
-    error = next(message for message in ws.sent if message.get("code") == "barge_in_unsupported")
-    assert error["session_id"] == "sid-native-barge-control-disabled"
-    assert ("sid-native-barge-control-disabled", "barge_in") not in engine.signals
+    assert not any(message.get("code") == "barge_in_unsupported" for message in ws.sent)
+    assert ("sid-native-barge-control-disabled", "barge_in") in engine.signals
 
 
 @pytest.mark.asyncio