Files
vllm-omni/tests/entrypoints/openai_api/test_duplex_handler.py
T

8134 lines
287 KiB
Python
Raw Normal View History

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import asyncio
import base64
import json
import struct
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
from starlette.websockets import WebSocketDisconnect
# Import the OpenAI entrypoints package at collection time, before pytest
# attaches its caplog handlers to the root logger. vllm's api_server
# transitively imports model_hosting_container_standards, whose import-time
# configure_root_logger() raises the level of every handler then attached to
# the root logger to ERROR. The serving runtime under test reaches this chain
# lazily (runtime_bridge -> entrypoints.openai.protocol.audio), so without
# this eager import the side effect fires mid-test and silences caplog for
# the remainder of the session. Before the capability predicate moved to
# vllm_omni.entrypoints.duplex.capability, this file imported the package at
# module scope implicitly.
import vllm_omni.entrypoints.openai # noqa: F401
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.engine.duplex.control_client import DuplexControlRequestError
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
from vllm_omni.engine.duplex.messages import DuplexFence, DuplexSessionLifecycleMessage
from vllm_omni.engine.duplex.runtime import duplex_resource_request_id
from vllm_omni.entrypoints.duplex import vad as realtime_vad
from vllm_omni.entrypoints.duplex.protocol import (
DuplexOverlapPolicy,
DuplexPlaybackCommitPolicy,
DuplexSession,
DuplexSessionConfig,
ResponseCreateOptions,
)
from vllm_omni.entrypoints.duplex.realtime_session import NativeRealtimeSessionProtocol
from vllm_omni.entrypoints.duplex.runtime_adapter import ServingRuntimeConfigError
from vllm_omni.entrypoints.duplex.serving import (
OmniDuplexSessionHandler,
should_enable_duplex_endpoint,
)
from vllm_omni.entrypoints.duplex.websocket import DuplexWebSocketActor
from vllm_omni.model_executor.models.minicpmo_4_5.duplex import (
MiniCPMO45NativeDuplexServingAdapter,
MiniCPMO45PcmAppendBuffer,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
minicpmo45_native_capabilities,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import (
MiniCPMO45DataPlaneContext,
MiniCPMO45DataPlaneSession,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import (
MiniCPMO45DuplexRuntimeExtension,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import (
MiniCPMO45ServingSessionState,
)
from vllm_omni.model_executor.models.personaplex.duplex.serving_adapter import (
PersonaPlexServingRuntimeAdapter,
)
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.outputs.duplex import attach_duplex_output_decision
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_native_input_append_supports_explicit_session_opt_out():
session = DuplexSession(
"sid-explicit-tts",
DuplexSessionConfig(model="test-model"),
capabilities=minicpmo45_native_capabilities(),
)
assert OmniDuplexSessionHandler._uses_native_input_append(session) is True
session.config.extra_body["native_duplex"] = False
assert OmniDuplexSessionHandler._uses_native_input_append(session) is False
class _ModelConfig:
model = "test-model"
class FakeEngineClient:
output_modalities = ["text", "audio"]
def __init__(
self,
*,
fail_open: bool = False,
fail_signal: bool = False,
fail_touch: bool = False,
fail_signal_events: set[str] | None = None,
fail_close: bool = False,
fail_abort: bool = False,
control_result: dict[str, object] | None = None,
open_result: dict[str, object] | None = None,
append_result: dict[str, object] | None = None,
append_results: list[dict[str, object]] | None = None,
collect_outputs: list[list[object]] | None = None,
collect_delay_s: float = 0.0,
signal_result: dict[str, object] | None = None,
close_result: dict[str, object] | None = None,
) -> None:
self.fail_open = fail_open
self.fail_signal = fail_signal
self.fail_touch = fail_touch
self.fail_signal_events = set(fail_signal_events or ())
self.fail_close = fail_close
self.fail_abort = fail_abort
self.control_result = control_result
self.open_result = open_result
self.append_result = append_result
self.append_results = list(append_results or [])
self.collect_outputs = list(collect_outputs or [])
self.collect_delay_s = collect_delay_s
self.signal_result = signal_result
self.close_result = close_result
self.opened: list[str] = []
self.opened_fences: list[DuplexFence | None] = []
self.appended: list[tuple[str, str, Any, bool]] = []
self.append_operation_ids: list[str | None] = []
self.appended_fences: list[DuplexFence | None] = []
self.opened_configs: list[dict[str, Any]] = []
self.opened_runtime_configs: list[dict[str, Any]] = []
self.signals: list[tuple[str, str]] = []
self.signal_fences: list[DuplexFence | None] = []
self.signal_next_fences: list[DuplexFence | None] = []
self.signal_session_configs: list[dict[str, Any] | None] = []
self.signal_runtime_configs: list[dict[str, Any] | None] = []
self.closed: list[tuple[str, str]] = []
self.touched: list[tuple[str, DuplexLeaseActivity]] = []
self.touch_fences: list[DuplexFence] = []
self.resumed: list[tuple[str, int]] = []
self.resume_fences: list[DuplexFence] = []
self.aborted: list[str] = []
self.abort_batches: list[list[str]] = []
self.internal_abort_batches: list[list[str]] = []
self.collected: list[tuple[str, int | None]] = []
self.duplex_lifecycle_events: asyncio.Queue[DuplexSessionLifecycleMessage] = asyncio.Queue()
async def open_duplex_session_async(
self,
session_id: str,
*,
session_mode: str = "duplex",
capabilities: dict[str, object] | None = None,
session_config: dict[str, object] | None = None,
runtime_config: dict[str, object] | None = None,
timeout: float | None = None,
fence: DuplexFence | None = None,
) -> Any:
del timeout
if self.fail_open:
raise RuntimeError("open failed")
self.opened.append(session_id)
self.opened_fences.append(fence)
self.opened_configs.append(dict(session_config or {}))
self.opened_runtime_configs.append(dict(runtime_config or {}))
return self.open_result if self.open_result is not None else self.control_result
async def append_duplex_input_async(
self,
session_id: str,
*,
mode: str,
payload: object,
operation_id: str | None = None,
final: bool = False,
timeout: float | None = None,
collect_outputs: bool = True,
fence: DuplexFence | None = None,
) -> Any:
del timeout, collect_outputs
self.appended.append((session_id, mode, payload, final))
self.append_operation_ids.append(operation_id)
self.appended_fences.append(fence)
if self.append_results:
return self.append_results.pop(0)
return self.append_result if self.append_result is not None else self.control_result
async def collect_duplex_data_plane_outputs_async(
self,
request_id: str,
*,
response_stage_id: int | None = None,
timeout: float | None = None,
) -> list[object]:
del timeout
self.collected.append((request_id, response_stage_id))
if self.collect_delay_s > 0:
await asyncio.sleep(self.collect_delay_s)
if not self.collect_outputs:
return []
return self.collect_outputs.pop(0)
async def signal_duplex_turn_async(
self,
session_id: str,
*,
event: str,
timeout: float | None = None,
fence: DuplexFence | None = None,
next_fence: DuplexFence | None = None,
session_config: dict[str, object] | None = None,
runtime_config: dict[str, object] | None = None,
) -> Any:
del timeout
if self.fail_signal or event in self.fail_signal_events:
raise RuntimeError("signal failed")
self.signals.append((session_id, event))
self.signal_fences.append(fence)
self.signal_next_fences.append(next_fence)
self.signal_session_configs.append(session_config)
self.signal_runtime_configs.append(runtime_config)
return self.signal_result if self.signal_result is not None else self.control_result
async def close_duplex_session_async(
self,
session_id: str,
*,
reason: str = "client_close",
timeout: float | None = None,
fence: DuplexFence | None = None,
) -> Any:
del timeout
if self.fail_close:
raise RuntimeError("close failed")
self.closed.append((session_id, reason))
return self.close_result if self.close_result is not None else self.control_result
async def touch_duplex_session_async(
self,
session_id: str,
*,
fence: DuplexFence,
activity: DuplexLeaseActivity,
timeout: float | None = None,
) -> dict[str, object]:
del timeout
if self.fail_touch:
raise RuntimeError("touch failed")
self.touched.append((session_id, activity))
self.touch_fences.append(fence)
return {
"operation": "touch",
"session_id": session_id,
"ok": True,
"stage_results": [
{
"stage_id": -1,
"replica_id": -1,
"result": {"supported": True, "lease_generation": 0},
}
],
}
async def resume_duplex_session_async(
self,
session_id: str,
*,
fence: DuplexFence,
expected_lease_generation: int,
timeout: float | None = None,
) -> dict[str, object]:
del timeout
self.resumed.append((session_id, expected_lease_generation))
self.resume_fences.append(fence)
return {
"operation": "resume",
"session_id": session_id,
"ok": True,
"stage_results": [
{
"stage_id": -1,
"replica_id": -1,
"result": {
"supported": True,
"lease_generation": expected_lease_generation + 1,
},
}
],
}
async def abort(self, request_ids: list[str]) -> None:
if self.fail_abort:
raise RuntimeError("abort failed")
assert isinstance(request_ids, list)
self.abort_batches.append(list(request_ids))
self.aborted.extend(request_ids)
async def _abort_internal_requests(self, request_ids: list[str]) -> None:
if self.fail_abort:
raise RuntimeError("abort failed")
assert isinstance(request_ids, list)
self.internal_abort_batches.append(list(request_ids))
self.aborted.extend(request_ids)
class FakeChatService:
duplex_serving_adapter_path = (
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
)
def __init__(self, engine_client: FakeEngineClient) -> None:
self.engine_client = engine_client
self.model_config = _ModelConfig()
self.seen_request_ids: list[str] = []
async def create_chat_completion(self, request, raw_request=None):
self.seen_request_ids.append(request.request_id)
async def _gen():
await asyncio.sleep(999)
yield "data: [DONE]\n\n"
return _gen()
def create_audio(self, audio_obj):
return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}")
def _test_data_plane() -> MiniCPMO45DataPlaneSession:
def encode_audio(
audio: object,
sample_rate_hz: int,
response_format: str,
speed: float | None,
) -> str | None:
del sample_rate_hz, response_format, speed
if audio is None:
return None
if hasattr(audio, "numel"):
return f"wav-{int(audio.numel())}"
return f"wav-{int(np.asarray(audio).size)}"
return MiniCPMO45DataPlaneSession(encode_audio)
def _data_plane_context(session: DuplexSession | None = None) -> MiniCPMO45DataPlaneContext:
if session is None:
return MiniCPMO45DataPlaneContext()
return MiniCPMO45DataPlaneContext(
epoch=session.epoch,
turn_id=session.turn_id,
active_response_turn_id=session.active_response_turn_id,
active_response_id=session.active_response_id,
auto_responds=bool(session.config.extra_body.get("auto_response", False)),
response_format=session.config.response_format,
speed=session.config.speed,
modalities=tuple(session.config.modalities),
)
def _project_data_plane(
data_plane: MiniCPMO45DataPlaneSession,
result: object,
*,
session: DuplexSession | None = None,
) -> list[dict[str, Any]]:
return list(data_plane.project(result, context=_data_plane_context(session)))
class TimedWebSocket:
def __init__(self, *, on_send=None, receive_timeout_s: float = 1.0):
self._q: asyncio.Queue[str | BaseException] = asyncio.Queue()
self.sent: list[dict[str, Any]] = []
self.accepted = False
self._on_send = on_send
self._receive_timeout_s = receive_timeout_s
self.query_params: dict[str, str] = {}
async def accept(self):
self.accepted = True
async def receive_text(self) -> str:
try:
item = await asyncio.wait_for(self._q.get(), timeout=self._receive_timeout_s)
except asyncio.TimeoutError as exc:
raise WebSocketDisconnect(code=1000) from exc
if isinstance(item, BaseException):
raise item
return item
async def send_json(self, data: dict[str, Any]):
self.sent.append(data)
if self._on_send is not None:
self._on_send(self, data)
def put(self, payload: dict[str, Any]) -> None:
self._q.put_nowait(json.dumps(payload))
async def close(self, code: int = 1000, reason: str | None = None) -> None:
del reason
self._q.put_nowait(WebSocketDisconnect(code=code))
def sent_types(self) -> list[str]:
return [m.get("type", "") for m in self.sent]
def test_native_realtime_protocol_emits_speak_once_per_response():
protocol = NativeRealtimeSessionProtocol({})
response_id = "resp-speak-once"
projected = protocol.encode_outbound_event(
{
"type": "response.created",
"response_id": response_id,
"modalities": ["audio", "text"],
}
)
projected.extend(
protocol.encode_outbound_event(
{
"type": "response.speak",
"response_id": response_id,
"text": "hello",
"model_speak": True,
}
)
)
projected.extend(
protocol.encode_outbound_event(
{
"type": "response.output_audio.delta",
"response_id": response_id,
"audio": base64.b64encode(b"\x00\x00").decode("ascii"),
"format": "pcm16",
"sample_rate_hz": 24000,
"model_speak": True,
}
)
)
assert [event["type"] for event in projected].count("response.speak") == 1
speak = next(event for event in projected if event["type"] == "response.speak")
assert "text" not in speak
assert speak["metadata"] == {"model_speak": True}
def test_native_realtime_protocol_speak_does_not_expose_internal_event():
protocol = NativeRealtimeSessionProtocol({})
projected = protocol.encode_outbound_event(
{
"type": "response.speak",
"response_id": "resp-curated-speak",
"session_id": "sid-curated-speak",
"epoch": 3,
"text": "transcript belongs on the delta channel",
"model_speak": True,
"data_plane_request_id": "internal-request-id",
"uses_model_runner_scheduler": True,
}
)
assert len(projected) == 1
projected[0].pop("event_id")
assert projected == [
{
"type": "response.speak",
"response_id": "resp-curated-speak",
"item_id": "item_resp-curated-speak",
"output_index": 0,
"content_index": 0,
"metadata": {
"session_id": "sid-curated-speak",
"epoch": 3,
"model_speak": True,
},
}
]
@pytest.mark.asyncio
async def test_native_realtime_protocol_drains_internal_conversation_item_control_event():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-delete"})
session_create = json.loads(await protocol.receive_internal_event_text(ws))
assert session_create["type"] == "session.create"
ws.put(
{
"type": "conversation.item.create",
"item": {
"id": "item-a",
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "hello"}],
},
}
)
assert json.loads(await protocol.receive_internal_event_text(ws)) == {
"type": "turn.signal",
"event": "conversation.item.create",
"payload": {
"item": {
"id": "item-a",
"object": "realtime.item",
"type": "message",
"status": "completed",
"role": "user",
"content": [{"type": "input_text", "text": "hello"}],
}
},
}
ws.put({"type": "conversation.item.delete", "item_id": "item-a"})
translated = json.loads(await protocol.receive_internal_event_text(ws))
assert translated == {
"type": "turn.signal",
"event": "conversation.item.delete",
"payload": {"item_id": "item-a"},
}
@pytest.mark.asyncio
async def test_native_realtime_protocol_conversation_item_create_commits_user_text():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-item-create"})
session_create = json.loads(await protocol.receive_internal_event_text(ws))
assert session_create["type"] == "session.create"
ws.put(
{
"type": "conversation.item.create",
"item": {
"id": "item-user-text",
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "hello"}],
},
}
)
translated = json.loads(await protocol.receive_internal_event_text(ws))
assert translated == {
"type": "turn.signal",
"event": "conversation.item.create",
"payload": {
"item": {
"id": "item-user-text",
"object": "realtime.item",
"type": "message",
"status": "completed",
"role": "user",
"content": [{"type": "input_text", "text": "hello"}],
}
},
}
@pytest.mark.asyncio
async def test_function_output_is_recorded_only_after_runtime_delivery():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
protocol.encode_outbound_event(
{
"type": "conversation.item.created",
"item": {
"id": "item-call",
"type": "function_call",
"call_id": "call-1",
"name": "weather",
"arguments": "{}",
"status": "completed",
},
}
)
event = {
"type": "conversation.item.create",
"item": {
"id": "item-output",
"type": "function_call_output",
"call_id": "call-1",
"output": '{"temperature":20}',
},
}
first = await protocol._to_duplex_event(event)
retry = await protocol._to_duplex_event(event)
assert first is not None
assert retry is not None
assert "item-output" not in protocol._conversation_items
delivered_item = first["payload"]["item"]
protocol.encode_outbound_event(
{
"type": "conversation.item.created",
"item": delivered_item,
}
)
assert protocol._conversation_items["item-output"] == delivered_item
assert await protocol._to_duplex_event(event) is None
assert ws.sent[-1]["error"]["code"] == "invalid_function_call_output"
@pytest.mark.asyncio
async def test_native_realtime_protocol_audio_commit_requires_non_empty_buffer():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-audio-commit"})
session_create = json.loads(await protocol.receive_internal_event_text(ws))
assert session_create["type"] == "session.create"
ws.put({"type": "input_audio_buffer.commit", "final": True})
with pytest.raises(WebSocketDisconnect):
await protocol.receive_internal_event_text(ws)
assert ws.sent[-1]["type"] == "error"
assert ws.sent[-1]["error"]["code"] == "input_audio_buffer_empty"
@pytest.mark.asyncio
async def test_native_realtime_protocol_audio_commit_does_not_auto_create_response():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-audio-commit-full"})
session_create = json.loads(await protocol.receive_internal_event_text(ws))
assert session_create["type"] == "session.create"
pcm = struct.pack("<2h", 1024, -1024)
ws.put({"type": "input_audio_buffer.append", "audio": base64.b64encode(pcm).decode("ascii"), "format": "pcm16"})
append_event = json.loads(await protocol.receive_internal_event_text(ws))
assert append_event["type"] == "input_audio_buffer.append"
ws.put({"type": "input_audio_buffer.commit", "final": True})
commit_event = json.loads(await protocol.receive_internal_event_text(ws))
assert commit_event["type"] == "input_audio_buffer.commit"
assert commit_event["response_create"] is False
@pytest.mark.asyncio
async def test_native_realtime_protocol_rejects_invalid_sample_rate_without_closing():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-invalid-rate"})
session_create = json.loads(await protocol.receive_internal_event_text(ws))
assert session_create["type"] == "session.create"
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(1),
"format": "pcm16",
"sample_rate_hz": 192_001,
}
)
ws.put({"type": "input_audio_buffer.clear"})
translated = json.loads(await protocol.receive_internal_event_text(ws))
assert translated == {"type": "input_audio_buffer.clear", "reason": "input_audio_buffer.clear"}
error = next(event for event in ws.sent if event.get("type") == "error")
assert error["error"]["code"] == "bad_event"
assert error["error"]["param"] == "sample_rate_hz"
@pytest.mark.asyncio
async def test_native_realtime_protocol_audio_clear_is_not_barge_in_cancel():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
ws.put({"type": "session.update", "model": "test-model", "session_id": "rt-audio-clear"})
session_create = json.loads(await protocol.receive_internal_event_text(ws))
assert session_create["type"] == "session.create"
ws.put({"type": "input_audio_buffer.clear"})
clear_event = json.loads(await protocol.receive_internal_event_text(ws))
assert clear_event == {"type": "input_audio_buffer.clear", "reason": "input_audio_buffer.clear"}
@pytest.mark.asyncio
async def test_native_realtime_protocol_preserves_input_turn_policy_hints():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
translated = await protocol._to_duplex_event(
{
"type": "input_audio_buffer.append",
"audio": "AAAA",
"format": "pcm_f32le",
"duration_ms": 240,
"vad": {"is_speech": True, "speech_probability": 0.9},
"overlap_action": "listen",
}
)
assert translated is not None
assert translated["duration_ms"] == 240
assert translated["vad"] == {"is_speech": True, "speech_probability": 0.9}
assert translated["overlap_action"] == "listen"
@pytest.mark.asyncio
async def test_native_realtime_protocol_rejects_server_vad_without_interrupt():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
translated = await protocol._to_duplex_event(
{
"type": "session.update",
"model": "test-model",
"session_id": "rt-turn-detection",
"turn_detection": {
"type": "server_vad",
"interrupt_response": False,
"silence_duration_ms": 900,
"threshold": 0.4,
},
}
)
assert translated is None
error = ws.sent[-1]
assert error["type"] == "error"
assert error["error"]["code"] == "unsupported_turn_detection"
assert "interrupt_response=false" in error["error"]["message"]
threshold_error = protocol._validate_realtime_turn_detection(
{"turn_detection": {"type": "server_vad", "threshold": 0.15}}
)
assert threshold_error == "turn_detection.threshold must be greater than 0.15 and at most 1"
@pytest.mark.asyncio
async def test_native_realtime_protocol_accepts_disabled_turn_detection():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
ws.put(
{
"type": "session.update",
"model": "test-model",
"session_id": "rt-no-turn-detection",
"turn_detection": None,
}
)
translated = json.loads(await protocol.receive_internal_event_text(ws))
assert translated["type"] == "session.create"
assert translated["session"]["extra_body"]["realtime_session_payload"]["turn_detection"] is None
@pytest.mark.asyncio
async def test_native_realtime_protocol_prefers_nested_vad_over_top_level():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
translated = await protocol._to_duplex_event(
{
"type": "session.update",
"model": "test-model",
"turn_detection": None,
"audio": {
"input": {
"turn_detection": {
"type": "server_vad",
}
}
},
}
)
assert translated["session"]["extra_body"]["realtime_session_payload"]["overlap_policy"] == "barge_in_on_speech"
@pytest.mark.asyncio
async def test_realtime_vad_edge_order_and_tiny_chunk_resampling(monkeypatch):
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
protocol._server_vad = realtime_vad.SileroStreamingVAD(realtime_vad.SileroVADConfig())
result = realtime_vad.StreamingVADResult(True, True, True, True)
monkeypatch.setattr(protocol._server_vad, "process_base64", lambda *args, **kwargs: result)
protocol._input_speech_started = True
protocol._hold_realtime_output_until_session_created = False
await protocol._to_duplex_event(
{"type": "input_audio_buffer.append", "audio": _pcm_f32_b64(1), "format": "pcm_f32le"}
)
assert [event["type"] for event in ws.sent] == [
"input_audio_buffer.speech_stopped",
"input_audio_buffer.speech_started",
]
ws.sent.clear()
protocol._input_speech_started = False
result = realtime_vad.StreamingVADResult(True, False, True, True)
await protocol._to_duplex_event({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"})
assert [event["type"] for event in ws.sent] == [
"input_audio_buffer.speech_started",
"input_audio_buffer.speech_stopped",
]
detector = realtime_vad.SileroStreamingVAD(realtime_vad.SileroVADConfig(), frame_scorer=lambda _: 0.0)
for _ in range(1024):
detector.process_base64(_pcm_f32_b64(1), fmt="pcm_f32le", sample_rate_hz=48_000)
assert detector._processed_samples + detector._pending.size == 341
@pytest.mark.asyncio
async def test_realtime_server_vad_runtime_failures_are_terminal(monkeypatch):
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
ws = TimedWebSocket()
ws.put(_server_vad_update("generic-vad", model="test-model", session_id="generic-vad"))
await handler.handle_session(ws, realtime_protocol=NativeRealtimeSessionProtocol({}))
assert ws.sent[-1]["error"]["code"] == "server_vad_requires_native_duplex"
monkeypatch.setattr(realtime_vad.SileroStreamingVAD, "process_base64", lambda *args, **kwargs: 1 / 0)
ws = TimedWebSocket()
create = _native_realtime_session_update("vad-inference-failure")
create["session"]["turn_detection"] = {"type": "server_vad"}
ws.put(create)
ws.put({"type": "input_audio_buffer.append", "audio": _pcm_f32_b64(1), "format": "pcm_f32le"})
await asyncio.wait_for(handler.handle_session(ws, realtime_protocol=NativeRealtimeSessionProtocol({})), timeout=2)
assert any(e.get("error", {}).get("code") == "realtime_input_failed" for e in ws.sent)
def test_native_duplex_handler_has_no_fixed_session_admission_cap():
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
client_config = DuplexSessionConfig(
extra_body={"max_duplex_sessions": 99, "duplex_max_sessions": 99},
)
assert not hasattr(handler, "_max_native_duplex_sessions")
assert not hasattr(handler, "_max_duplex_sessions")
assert client_config.extra_body["max_duplex_sessions"] == 99
def test_native_response_options_ignore_private_runtime_config():
session = DuplexSession(
session_id="sid-response-private-config",
config=DuplexSessionConfig(),
capabilities=minicpmo45_native_capabilities(),
)
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
handler._apply_response_create_options(
session,
{
"extra_body": {
"duplex_stage_max_tokens": {"0": 999},
"client_trace_id": "trace-1",
}
},
)
session.begin_response()
assert "duplex_stage_max_tokens" not in session.config.extra_body
assert "client_trace_id" not in session.config.extra_body
assert session.response_config.extra_body["client_trace_id"] == "trace-1"
@pytest.mark.parametrize(
"payload",
[
{"instructions": "not wired"},
{"voice": "not-wired"},
{"temperature": 0.2},
{"max_output_tokens": 8},
{"tools": [{"type": "function", "name": "lookup"}]},
],
)
def test_native_response_create_rejects_unwired_generation_options(payload):
session = DuplexSession(
session_id="sid-response-unwired-options",
config=DuplexSessionConfig(),
capabilities=minicpmo45_native_capabilities(),
)
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
error = handler._apply_response_create_options(session, payload)
assert error == "unsupported_native_response_options"
session.begin_response()
assert session.response_config == session.config
def test_chat_fallback_excludes_realtime_response_metadata_from_model_request():
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-response-protocol-metadata",
config=DuplexSessionConfig(),
)
session.reserve_response_options(
ResponseCreateOptions(
extra_body={
"user": "trace-1",
"realtime_response_conversation": "none",
"realtime_response_metadata": {"source": "test"},
"realtime_response_prompt": {"id": "prompt-1"},
"realtime_response_tools": [
{
"type": "function",
"function": {"name": "lookup", "parameters": {}},
}
],
"realtime_response_tool_choice": "auto",
}
)
)
session.begin_response()
request = handler._build_chat_request(session, "request-1")
created = handler._response_created_payload(session, session.active_response_id or "response-1", epoch=0)
dumped = request.model_dump()
assert dumped["user"] == "trace-1"
assert dumped["tools"][0]["function"]["name"] == "lookup"
assert dumped["tool_choice"] == "auto"
assert not any(key.startswith("realtime_response_") for key in dumped)
assert created["conversation"] == "none"
assert created["metadata"] == {"source": "test"}
assert created["prompt"] == {"id": "prompt-1"}
assert session.response_config.extra_body["realtime_response_conversation"] == "none"
@pytest.mark.asyncio
async def test_native_response_create_surfaces_unwired_generation_options_error():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-response-unwired-options-event"))
ws.put(
{
"type": "response.create",
"response": {"instructions": "not wired"},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
error = next(message for message in ws.sent if message.get("type") == "error")
assert error["code"] == "unsupported_native_response_options"
assert "response_create_without_input" not in {message.get("code") for message in ws.sent}
@pytest.mark.asyncio
async def test_realtime_session_update_preserves_tools_and_metadata():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-rt-update-fields"))
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {
"tools": [{"type": "function", "name": "lookup"}],
"tool_choice": "auto",
"metadata": {"demo": "yes"},
},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
updated = next(m for m in ws.sent if m.get("type") == "session.updated")
session = updated["session"]
assert session["tools"] == [{"type": "function", "name": "lookup"}]
assert session["tool_choice"] == "auto"
assert session["metadata"] == {"demo": "yes"}
assert "turn_detection" not in session
runtime_config = engine.signal_session_configs[-1]
assert runtime_config is not None
assert runtime_config["extra_body"]["realtime_tools"] == [{"type": "function", "name": "lookup"}]
assert runtime_config["extra_body"]["realtime_metadata"] == {"demo": "yes"}
@pytest.mark.asyncio
async def test_native_realtime_protocol_non_speech_append_does_not_emit_speech_started():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
translated = await protocol._to_duplex_event(
{
"type": "input_audio_buffer.append",
"audio": "AAAA",
"format": "pcm_f32le",
"vad": {"is_speech": False},
}
)
assert translated is not None
assert translated["vad"] == {"is_speech": False}
assert not any(event.get("type") == "input_audio_buffer.speech_started" for event in ws.sent)
@pytest.mark.asyncio
async def test_native_session_update_rebuilds_server_runtime_policy():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-runtime-update"))
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"temperature": 0.2, "max_output_tokens": 7},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert "session.updated" in ws.sent_types()
public_config = engine.signal_session_configs[-1]
runtime_config = engine.signal_runtime_configs[-1]
assert public_config is not None
assert runtime_config is not None
assert "duplex_stage_sampling_params" not in public_config["extra_body"]
assert runtime_config["duplex_stage_max_tokens"]["0"] == 7
assert runtime_config["duplex_stage_sampling_params"]["0"]["temperature"] == 0.2
assert runtime_config["duplex_stage_sampling_params"]["0"]["top_k"] == 20
assert runtime_config["duplex_stage_sampling_params"]["0"]["top_p"] == 0.8
@pytest.mark.asyncio
async def test_native_session_update_rejects_client_runtime_config():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-runtime-update-reject"))
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {
"extra_body": {"duplex_stage_max_tokens": {"0": 999}},
},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
error = next(message for message in ws.sent if message.get("type") == "error")
assert error["code"] == "invalid_duplex_runtime_config"
assert ("sid-runtime-update-reject", "session.update") not in engine.signals
@pytest.mark.asyncio
async def test_minicpmo_native_session_update_requires_ref_audio_before_enabling_audio():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-runtime-update-ref-required", modalities=["text"]))
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"modalities": ["text", "audio"]},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
error = next(message for message in ws.sent if message.get("type") == "error")
assert error["code"] == "ref_audio_required"
assert "session.updated" not in ws.sent_types()
assert ("sid-runtime-update-ref-required", "session.update") not in engine.signals
def test_personaplex_candidate_update_does_not_leak_minicpmo_ref_audio_check():
handler = OmniDuplexSessionHandler(
chat_service=SimpleNamespace(engine_client=SimpleNamespace()),
serving_runtime_adapter=PersonaPlexServingRuntimeAdapter(lambda *_: None),
)
session = DuplexSession(
session_id="sid-personaplex-voice-change",
config=DuplexSessionConfig(modalities=["audio"], voice="NATF2.pt"),
capabilities=PersonaPlexServingRuntimeAdapter.capabilities(max_sessions=1),
)
candidate_config = DuplexSessionConfig(modalities=["audio"], voice="OtherVoice.pt")
error = handler._runtime_session_candidate_update_error(session, candidate_config)
assert error is None
@pytest.mark.asyncio
async def test_minicpmo_native_session_update_rejects_instructions_after_native_context_locked():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-instructions-locked", modalities=["text"]))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"duration_ms": 1000,
"is_speech": True,
}
)
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"instructions": "You are now a pirate."},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
error = next(message for message in ws.sent if message.get("type") == "error")
assert error["code"] == "instructions_update_unsupported"
assert "session.updated" not in ws.sent_types()
assert ("sid-instructions-locked", "session.update") not in engine.signals
@pytest.mark.asyncio
async def test_minicpmo_native_session_update_allows_unchanged_instructions_after_lock():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-instructions-unchanged", modalities=["text"]))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"duration_ms": 1000,
"is_speech": True,
}
)
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"instructions": "You are a concise assistant.", "temperature": 0.3},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert "session.updated" in ws.sent_types()
assert not any(
message.get("type") == "error" and message.get("code") == "instructions_update_unsupported"
for message in ws.sent
)
@pytest.mark.asyncio
async def test_minicpmo_native_session_update_rejects_instructions_before_first_real_append():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-buffered-before-lock", modalities=["text"]))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(3200, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"duration_ms": 200,
"is_speech": True,
}
)
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {
"instructions": (
"You are now a swashbuckling pirate captain who speaks entirely in "
"nautical slang, references buried treasure constantly, and never "
"breaks character no matter what the user asks."
)
},
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True, "response_create": True})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
error = next(message for message in ws.sent if message.get("type") == "error")
assert error["code"] == "instructions_update_unsupported"
assert "session.updated" not in ws.sent_types()
assert engine.appended, "the commit must still flush the buffered chunk despite the rejected update"
@pytest.mark.asyncio
async def test_minicpmo_native_session_update_rejects_native_duplex_mode_flip():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-duplex-flag-flip", modalities=["text"]))
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"extra_body": {"native_duplex": False}},
}
)
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"instructions": "You are now a pirate."},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
errors = [message for message in ws.sent if message.get("type") == "error"]
assert [error["code"] for error in errors] == [
"native_duplex_mode_update_unsupported",
"instructions_update_unsupported",
]
assert "session.updated" not in ws.sent_types()
def test_native_realtime_protocol_audio_delta_preserves_sample_rate_hz():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
payloads = protocol._from_duplex_event(
{
"type": "response.output_audio.delta",
"response_id": "resp-a",
"audio": "AAAA",
"format": "pcm",
"sample_rate_hz": 24000,
}
)
audio_events = [
payload for payload in payloads if payload["type"] in {"response.output_audio.delta", "response.audio.delta"}
]
assert {payload["type"] for payload in audio_events} == {"response.audio.delta"}
assert {payload["format"] for payload in audio_events} == {"pcm16"}
assert {payload["sample_rate_hz"] for payload in audio_events} == {24000}
def test_native_realtime_protocol_ignores_removed_legacy_event_switches():
ws = TimedWebSocket()
ws.query_params = {
"legacy_audio_events": "true",
"vllm_omni_legacy_events": "true",
"output_audio_events": "true",
"vllm_omni_output_audio_events": "true",
}
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
payloads = protocol._from_duplex_event(
{
"type": "response.output_audio.delta",
"response_id": "resp-canonical",
"audio": "AAAA",
"text": "hello",
"format": "pcm",
"sample_rate_hz": 24000,
"end_of_turn": True,
}
)
event_types = {payload["type"] for payload in payloads}
assert "response.audio.delta" in event_types
assert "response.audio.done" in event_types
assert "response.audio_transcript.delta" in event_types
assert "response.audio_transcript.done" in event_types
assert "response.output_audio.delta" not in event_types
assert "response.output_audio.done" not in event_types
assert "response.output_audio_transcript.delta" not in event_types
assert "response.output_audio_transcript.done" not in event_types
assert "response.output_item.created" not in event_types
assert "response.text.delta" not in event_types
def test_native_realtime_protocol_emits_terminal_audio_transcript_events():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
payloads = protocol._from_duplex_event(
{
"type": "response.output_audio.delta",
"response_id": "resp-a",
"audio": "AAAA",
"text": "hello",
"format": "pcm",
"sample_rate_hz": 24000,
"end_of_turn": True,
}
)
by_type = {payload["type"]: payload for payload in payloads}
assert "response.audio.done" in by_type
assert "response.audio_transcript.delta" in by_type
assert "response.audio_transcript.done" in by_type
assert by_type["response.content_part.done"]["part"]["transcript"] == "hello"
assert by_type["response.output_item.done"]["item"]["object"] == "realtime.item"
assert by_type["response.done"]["response"]["output"][0]["object"] == "realtime.item"
def test_native_realtime_protocol_terminal_transcript_equals_joined_deltas():
protocol = NativeRealtimeSessionProtocol(TimedWebSocket()) # type: ignore[arg-type]
response_id = "resp-transcript-contract"
events: list[dict[str, object]] = []
for text, end_of_turn in (("It's Canberra.", False), (" Next question.", True)):
events.extend(
protocol._from_duplex_event(
{
"type": "response.output_audio.delta",
"response_id": response_id,
"audio": "AAAA",
"text": text,
"format": "pcm",
"sample_rate_hz": 24000,
"end_of_turn": end_of_turn,
}
)
)
transcript = "".join(str(event["delta"]) for event in events if event["type"] == "response.audio_transcript.delta")
done = [str(event["transcript"]) for event in events if event["type"] == "response.audio_transcript.done"]
assert transcript == "It's Canberra. Next question."
assert done == [transcript]
def test_native_realtime_protocol_updates_in_progress_item_for_audio_truncate():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol._from_duplex_event({"type": "response.created", "response_id": "resp-truncate"})
protocol._from_duplex_event(
{
"type": "response.output_audio.delta",
"response_id": "resp-truncate",
"audio": "AAAA",
"text": "hello",
"format": "pcm",
"sample_rate_hz": 24000,
"audio_duration_ms": 100,
"audio_text_marks": [{"text_chars": 5, "audio_end_ms": 100}],
}
)
item = protocol._conversation_items["item_resp-truncate"]
assert item["content"][0]["type"] == "output_audio"
assert item["content"][0]["transcript"] == "hello"
assert item["content"][0]["audio_duration_ms"] == 100
translated = asyncio.run(
protocol._to_duplex_event(
{
"type": "conversation.item.truncate",
"item_id": "item_resp-truncate",
"content_index": 0,
"audio_end_ms": 50,
}
)
)
assert translated is None
assert protocol._conversation_items["item_resp-truncate"]["content"][0]["transcript"] == "he"
@pytest.mark.asyncio
async def test_native_realtime_protocol_rejects_truncate_for_user_item():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
protocol._conversation_items["item-user"] = {
"id": "item-user",
"object": "realtime.item",
"type": "message",
"role": "user",
"status": "completed",
"content": [{"type": "input_audio", "transcript": "hello"}],
}
translated = await protocol._to_duplex_event(
{
"type": "conversation.item.truncate",
"item_id": "item-user",
"content_index": 0,
"audio_end_ms": 10,
}
)
assert translated is None
error = next(event for event in ws.sent if event.get("type") == "error")
assert error["error"]["code"] == "bad_event"
def test_native_realtime_protocol_response_done_emits_audio_done_before_terminal_events():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol._from_duplex_event({"type": "response.created", "response_id": "resp-done"})
protocol._from_duplex_event(
{
"type": "response.output_audio.delta",
"response_id": "resp-done",
"audio": "AAAA",
"format": "pcm",
"sample_rate_hz": 24000,
}
)
payloads = protocol._from_duplex_event({"type": "response.done", "response_id": "resp-done"})
event_types = [payload["type"] for payload in payloads]
assert event_types.index("response.audio.done") < event_types.index("response.content_part.done")
assert event_types.index("response.content_part.done") < event_types.index("response.output_item.done")
assert event_types.index("conversation.item.done") < event_types.index("response.done")
assert event_types.index("response.output_item.done") < event_types.index("response.done")
def test_native_realtime_protocol_audio_cancelled_uses_active_response_id():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol._from_duplex_event({"type": "response.created", "response_id": "resp-cancel"})
payloads = protocol._from_duplex_event(
{
"type": "audio.cancelled",
"reason": "barge_in",
"committed_ms": 0,
}
)
done = [payload for payload in payloads if payload.get("type") == "response.done"]
assert len(done) == 1
assert done[0]["response_id"] == "resp-cancel"
assert done[0]["response"]["id"] == "resp-cancel"
def test_native_realtime_protocol_audio_cancelled_does_not_reopen_completed_response():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol._from_duplex_event({"type": "response.created", "response_id": "resp-complete"})
protocol._from_duplex_event(
{
"type": "response.output_audio.delta",
"response_id": "resp-complete",
"audio": "AAAA",
"format": "pcm",
"sample_rate_hz": 24000,
}
)
protocol._from_duplex_event({"type": "response.done", "response_id": "resp-complete"})
payloads = protocol._from_duplex_event(
{
"type": "audio.cancelled",
"reason": "session_close",
"committed_ms": 999999,
}
)
assert [payload.get("type") for payload in payloads] == []
def test_native_audio_text_marks_are_normalized_to_session_cumulative_offsets():
marks = OmniDuplexSessionHandler._normalize_native_audio_text_marks(
[{"text_chars": 3, "audio_end_ms": 200}],
audio_offset_ms=800,
text_offset_chars=5,
)
assert marks == [{"text_chars": 8, "audio_end_ms": 1000}]
def test_duplex_session_playback_commit_uses_multi_delta_audio_text_marks():
session = DuplexSession(
session_id="sid-marks",
config=DuplexSessionConfig(playback_commit_policy=DuplexPlaybackCommitPolicy.ACK_ONLY.value),
)
session.begin_response()
session.append_assistant_text("hello ")
session.mark_audio_sent(1000, text_chars=6)
session.append_assistant_text("world")
session.mark_audio_sent(2000, text_chars=11)
session.acknowledge_playback(played_ms=1500, committed_ms=1500)
committed = session.end_response(commit_text=True)
assert committed == {"role": "assistant", "content": "hello wo"}
assert session.history == (committed,)
@pytest.mark.asyncio
async def test_duplex_session_actor_preserves_wire_order_before_control():
ws = TimedWebSocket()
actor = DuplexWebSocketActor(ws)
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "AAAA"})
await actor.enqueue_event({"type": "response.cancel"})
first = await actor.next_event()
second = await actor.next_event()
assert first["type"] == "input_audio_buffer.append"
assert second["type"] == "response.cancel"
def _pcm_f32_b64(samples: int, *, value: float = 0.05) -> str:
return base64.b64encode(struct.pack(f"<{samples}f", *([value] * samples))).decode("ascii")
def _pcm16_b64(samples: int = 1, *, value: int = 1000) -> str:
return base64.b64encode(struct.pack(f"<{samples}h", *([value] * samples))).decode("ascii")
def _session_create(session_id: str = "duplex-test") -> dict[str, Any]:
return {
"type": "session.create",
"session_id": session_id,
"session": {
"model": "test-model",
"modalities": ["text", "audio"],
"idle_timeout_s": 1,
},
}
def _native_session_create(
session_id: str = "duplex-native",
*,
modalities: list[str] | None = None,
) -> dict[str, Any]:
event = _session_create(session_id)
event["session"]["model"] = "openbmb/MiniCPM-o-4_5"
event["session"]["modalities"] = list(modalities or ["text"])
event["session"]["instructions"] = "You are a concise assistant."
event["session"]["extra_body"] = {"native_duplex": True}
return event
def _native_realtime_session_update(
session_id: str,
*,
modalities: list[str] | None = None,
) -> dict[str, Any]:
return {
"type": "session.update",
"session": {
"session_id": session_id,
"model": "openbmb/MiniCPM-o-4_5",
"modalities": list(modalities or ["text"]),
"instructions": "You are a concise assistant.",
"idle_timeout_s": 1,
"extra_body": {"native_duplex": True},
},
}
def _server_vad_update(event_id: str, **session: object) -> dict[str, object]:
return {
"type": "session.update",
"event_id": event_id,
"session": {"turn_detection": {"type": "server_vad"}, **session},
}
def test_native_stage0_request_id_matches_engine_after_same_session_id_reopen():
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
first = handler._registry.create(session_id="sid-reused")
handler._registry.close(first.session_id)
reopened = handler._registry.create(session_id="sid-reused")
expected = duplex_resource_request_id(
DuplexFence(
reopened.session_id,
epoch=reopened.epoch,
incarnation=reopened.incarnation,
),
"stage0",
)
assert reopened.incarnation == 1
assert handler._native_stage0_request_id(reopened, reopened.epoch) == expected
def _native_audio_payload(
*,
samples: int = 16000,
value: float = 0.05,
is_speech: bool | None = True,
force_listen: bool | None = None,
) -> dict[str, Any]:
payload: dict[str, Any] = {
"type": "audio",
"audio": _pcm_f32_b64(samples, value=value),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
if is_speech is not None:
payload["is_speech"] = is_speech
if force_listen is not None:
payload["force_listen"] = force_listen
return payload
def _auto_response_context(
session_id: str,
*,
playback_active: bool = False,
) -> tuple[OmniDuplexSessionHandler, DuplexSession]:
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id=session_id,
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
if playback_active:
session.config.playback_commit_policy = DuplexPlaybackCommitPolicy.ACK_ONLY.value
session.begin_response()
session.mark_audio_sent(duration_ms=1000)
return handler, session
def _install_direct_silence_scheduler(
handler: OmniDuplexSessionHandler,
session: DuplexSession,
) -> None:
native = handler._runtime_session_state(session)
async def _schedule(payload: object, **kwargs: Any) -> bool:
if handler._native_silence_continuation_is_stale(
session,
request_id=kwargs["request_id"],
response_id=kwargs["response_id"],
response_owned=kwargs["response_owned"],
expected_epoch=kwargs["expected_epoch"],
expected_incarnation=kwargs["expected_incarnation"],
expected_model_turn_id=kwargs["expected_model_turn_id"],
):
return False
append_ok, _ = await handler._append_runtime_input(
session,
payload,
final=False,
send_json=kwargs["send_json"],
mode="append_audio_chunk",
expected_epoch=kwargs["expected_epoch"],
)
return append_ok
native.silence_continuation_scheduler = _schedule
def test_minicpmo_pcm_append_buffer_flush_preserves_accumulated_speech_marker():
buffer = MiniCPMO45PcmAppendBuffer()
payload = _native_audio_payload(samples=8000)
assert buffer.append(payload, chunk_period_ms=1000) is None
flushed = buffer.flush(chunk_period_ms=1000)
assert flushed is not None
assert flushed["is_speech"] is True
def test_minicpmo_pcm_append_buffer_drops_serving_new_user_turn_marker():
buffer = MiniCPMO45PcmAppendBuffer()
first = _native_audio_payload(samples=8000)
first.update(
new_user_turn=True,
new_user_turn_prefix_variant="clean_response_done",
force_speak=True,
)
second = _native_audio_payload(samples=8000)
assert buffer.append(first, chunk_period_ms=1000) is None
emitted = buffer.append(second, chunk_period_ms=1000)
assert emitted is not None
assert emitted["is_speech"] is True
assert "new_user_turn" not in emitted
assert "new_user_turn_prefix_variant" not in emitted
assert "force_speak" not in emitted
def test_minicpmo_merge_native_audio_payloads_preserves_speech_marker():
first = _native_audio_payload(samples=8000)
second = _native_audio_payload(samples=8000, value=0.0, is_speech=False)
merged = OmniDuplexSessionHandler._merge_native_audio_payloads(first, second)
assert merged["is_speech"] is True
def test_minicpmo_merge_drops_serving_new_user_turn_marker():
first = _native_audio_payload(samples=8000)
first.update(
new_user_turn=True,
new_user_turn_prefix_variant="clean_response_done",
force_speak=True,
)
second = _native_audio_payload(samples=8000, is_speech=False)
merged = OmniDuplexSessionHandler._merge_native_audio_payloads(first, second)
assert merged["is_speech"] is True
assert "new_user_turn" not in merged
assert "new_user_turn_prefix_variant" not in merged
assert "force_speak" not in merged
@pytest.mark.asyncio
async def test_minicpmo_clear_continuation_does_not_cancel_pending_silence_task():
async def _pending() -> bool:
await asyncio.sleep(3600)
return True
native = MiniCPMO45ServingSessionState()
task = asyncio.create_task(_pending())
native.continuation_owner_id = "owner"
native.continuation_units = 1
native.pending_silence_task = task
native.pending_silence_owner_id = "owner"
native.clear_continuation()
assert native.continuation_owner_id is None
assert native.continuation_units == 0
assert native.pending_silence_task is None
assert native.pending_silence_owner_id is None
assert not task.cancelled()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
def test_auto_response_playback_overlap_keeps_model_owned_listen_speak_decision():
handler, session = _auto_response_context("sid-auto-silent-overlap", playback_active=True)
session.mark_audio_sent(duration_ms=1000)
silent_payload = _native_audio_payload(value=0.0, is_speech=None, force_listen=False)
speech_payload = _native_audio_payload(is_speech=None, force_listen=False)
assert handler._should_force_listen_for_auto_response_overlap(session, {}, silent_payload) is False
assert handler._should_force_listen_for_auto_response_overlap(session, {}, speech_payload) is False
assert (
handler._should_force_listen_for_auto_response_overlap(
session,
{"force_listen": True},
speech_payload,
)
is True
)
assert (
handler._should_force_listen_for_auto_response_overlap(
session,
{"force_barge_in": True},
speech_payload,
)
is False
)
session.acknowledge_playback(played_ms=1000, committed_ms=1000)
assert handler._should_force_listen_for_auto_response_overlap(session, {}, speech_payload) is False
def test_legacy_auto_overlap_policy_falls_back_to_listen_only():
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-auto-policy-fallback",
config=DuplexSessionConfig(
overlap_policy=DuplexSessionConfig._normalize_overlap_policy("auto"),
),
)
session.capabilities = minicpmo45_native_capabilities()
decision = handler._overlap_decision(
session,
{"duration_ms": session.config.overlap_barge_in_ms, "is_speech": True},
_native_audio_payload(is_speech=True),
)
assert session.config.overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
assert decision["action"] == "listen"
assert decision["reason"] == "policy_listen_only"
def test_auto_response_overlap_silence_advances_model_unit_and_preserves_realtime_input():
handler, session = _auto_response_context(
"sid-auto-silent-wire-buffer",
playback_active=True,
)
decision = handler._overlap_decision(
session,
{"is_speech": False},
_native_audio_payload(samples=3200, is_speech=False),
)
assert decision["action"] == "listen"
assert decision["buffer_audio"] is True
assert decision["defer_runtime_append"] is False
assert decision["force_listen"] is False
assert decision["preserve_realtime_input"] is True
def test_auto_response_playback_overlap_admits_model_units_and_tracks_speech():
handler, session = _auto_response_context("sid-auto-deferred-overlap", playback_active=True)
payload = _native_audio_payload(samples=8640)
decision = handler._overlap_decision(
session,
{"duration_ms": 540, "is_speech": True},
payload,
)
assert decision["action"] == "listen"
assert decision["defer_runtime_append"] is False
assert decision["force_listen"] is False
assert decision["buffer_audio"] is True
assert session.overlap_speech_ms == 540
@pytest.mark.asyncio
async def test_auto_response_overlap_exact_unit_commit_does_not_block_or_replay_next_unit():
class ModelUnitEngine(FakeEngineClient):
def __init__(self) -> None:
super().__init__()
self.append_started = asyncio.Event()
self.active_response_ids_at_append: list[str | None] = []
self.response_done_seen_at_append = False
self.session: DuplexSession | None = None
self.websocket: TimedWebSocket | None = None
async def append_duplex_input_async(self, session_id: str, **kwargs):
assert self.session is not None
assert self.websocket is not None
self.active_response_ids_at_append.append(self.session.active_response_id)
self.response_done_seen_at_append = "response.done" in self.websocket.sent_types()
kwargs.pop("expected_epoch", None)
result = await super().append_duplex_input_async(session_id, **kwargs)
self.append_started.set()
return result
engine = ModelUnitEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
engine.websocket = ws
create = _native_session_create("sid-auto-overlap-model-unit")
create["session"]["extra_body"]["auto_response"] = True
ws.put(create)
handler_task = asyncio.create_task(handler.handle_session(ws))
try:
for _ in range(100):
session = handler._registry.get("sid-auto-overlap-model-unit")
if session is not None and "session.created" in ws.sent_types():
break
await asyncio.sleep(0.01)
else:
pytest.fail("native duplex session did not open")
engine.session = session
session.config.playback_commit_policy = DuplexPlaybackCommitPolicy.ACK_ONLY.value
response_id = session.begin_response()
session.mark_audio_sent(duration_ms=2000)
request_id = handler._native_stage0_request_id(session, session.epoch)
session.bind_request(request_id)
expected_fence = DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=session.turn_id,
incarnation=session.incarnation,
)
chunk = {
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(3200, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"duration_ms": 200,
"is_speech": True,
}
for _ in range(4):
ws.put(chunk)
await asyncio.sleep(0.05)
assert engine.appended == []
ws.put(chunk)
await asyncio.wait_for(engine.append_started.wait(), timeout=0.5)
ws.put({"type": "input_audio_buffer.commit", "final": True})
for _ in range(100):
if "input.committed" in ws.sent_types():
break
await asyncio.sleep(0.01)
else:
pytest.fail("exact model unit input did not commit")
engine.append_started.clear()
for _ in range(5):
ws.put(chunk)
await asyncio.wait_for(engine.append_started.wait(), timeout=0.5)
await asyncio.sleep(0.05)
assert "error" not in ws.sent_types()
finally:
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert len(engine.appended) == 2
for appended_session_id, mode, payload, final in engine.appended:
assert appended_session_id == session.session_id
assert mode == "append_audio_chunk"
assert final is False
assert isinstance(payload, dict)
assert np.frombuffer(base64.b64decode(payload["audio"]), dtype="<f4").shape == (16000,)
assert payload.get("force_listen") is not True
assert engine.appended_fences == [expected_fence, expected_fence]
assert engine.active_response_ids_at_append == [response_id, response_id]
assert engine.response_done_seen_at_append is False
assert not any(event in {"barge_in", "input.cancel"} for _, event in engine.signals)
assert ws.sent_types().count("response.speak") <= 1
@pytest.mark.asyncio
async def test_stale_terminal_event_does_not_reset_current_session_overlap():
handler, session = _auto_response_context("sid-stale-terminal")
session.epoch = 2
session.accumulate_overlap_speech(540)
actor = DuplexWebSocketActor(TimedWebSocket(), current_epoch=lambda: session.epoch)
native = MiniCPMO45ServingSessionState()
accepted, deferred = await handler._apply_outbound_session_event(
{"type": "response.done", "epoch": 1},
session=session,
actor=actor,
native=native,
realtime_protocol=None,
)
assert accepted is False
assert deferred is None
assert session.overlap_speech_ms == 540
@pytest.mark.asyncio
async def test_current_terminal_event_resets_session_owned_overlap():
handler, session = _auto_response_context("sid-current-terminal")
session.accumulate_overlap_speech(320)
actor = DuplexWebSocketActor(TimedWebSocket(), current_epoch=lambda: session.epoch)
native = MiniCPMO45ServingSessionState()
accepted, deferred = await handler._apply_outbound_session_event(
{"type": "response.done", "epoch": session.epoch},
session=session,
actor=actor,
native=native,
realtime_protocol=None,
)
assert accepted is True
assert deferred is None
assert session.overlap_speech_ms == 0
@pytest.mark.asyncio
async def test_response_listen_terminal_does_not_drive_serving_generation_state():
handler, session = _auto_response_context("sid-listen-terminal")
actor = DuplexWebSocketActor(TimedWebSocket(), current_epoch=lambda: session.epoch)
native = MiniCPMO45ServingSessionState()
accepted, deferred = await handler._apply_outbound_session_event(
{"type": "response.listen", "epoch": session.epoch},
session=session,
actor=actor,
native=native,
realtime_protocol=None,
)
assert accepted is True
assert deferred is None
assert session.turn_id == 0
@pytest.mark.asyncio
async def test_stale_audio_cancelled_does_not_reset_current_session_overlap():
handler, session = _auto_response_context("sid-stale-audio-cancelled")
session.epoch = 3
session.accumulate_overlap_speech(640)
actor = DuplexWebSocketActor(TimedWebSocket(), current_epoch=lambda: session.epoch)
native = MiniCPMO45ServingSessionState()
accepted, deferred = await handler._apply_outbound_session_event(
{"type": "audio.cancelled", "epoch": 2},
session=session,
actor=actor,
native=native,
realtime_protocol=None,
)
assert accepted is False
assert deferred is None
assert session.overlap_speech_ms == 640
def test_auto_response_post_response_silence_remains_model_owned_input():
handler, session = _auto_response_context("sid-auto-waiting-silence", playback_active=True)
silent_payload = _native_audio_payload(value=0.0, is_speech=False, force_listen=False)
speech_payload = _native_audio_payload(force_listen=False)
assert handler._assistant_playback_active(session) is True
assert handler._should_force_listen_for_auto_response_overlap(session, {}, silent_payload) is False
assert handler._should_force_listen_for_auto_response_overlap(session, {}, speech_payload) is False
@pytest.mark.asyncio
async def test_auto_response_terminal_advances_open_realtime_input_without_commit():
handler, session = _auto_response_context("sid-auto-realtime-open-input", playback_active=True)
session.capabilities = minicpmo45_native_capabilities()
response_id = session.active_response_id
assert response_id is not None
native = handler._runtime_session_state(session)
native.input_since_commit = True
native.speech_since_commit = True
session.accumulate_overlap_speech(800)
assert (
native.audio_buffer.append(
_native_audio_payload(samples=8000),
chunk_period_ms=session.capabilities.chunk_period_ms or 1000,
)
is None
)
session.end_response(commit_text=False, preserve_request=True)
allowed, promoted = await handler._apply_outbound_session_event(
{
"type": "response.done",
"response_id": response_id,
"epoch": session.epoch,
"status": "completed",
},
session=session,
actor=DuplexWebSocketActor(TimedWebSocket()),
native=native,
realtime_protocol=NativeRealtimeSessionProtocol({}),
)
assert allowed is True
assert promoted is None
assert native.audio_buffer.has_pending()
assert native.committed_audio_payload is None
assert native.deferred_response_create is False
assert native.input_since_commit is True
assert native.speech_since_commit is True
next_speech = _native_audio_payload(samples=8000)
assert "new_user_turn" not in next_speech
@pytest.mark.asyncio
async def test_auto_response_terminal_allows_next_model_unit_while_playback_drains():
handler, session = _auto_response_context(
"sid-auto-realtime-open-before-overlap",
playback_active=True,
)
session.capabilities = minicpmo45_native_capabilities()
response_id = session.active_response_id
assert response_id is not None
native = handler._runtime_session_state(session)
native.input_since_commit = True
native.speech_since_commit = True
session.end_response(commit_text=False, preserve_request=True)
allowed, promoted = await handler._apply_outbound_session_event(
{
"type": "response.done",
"response_id": response_id,
"epoch": session.epoch,
"status": "completed",
},
session=session,
actor=DuplexWebSocketActor(TimedWebSocket()),
native=native,
realtime_protocol=NativeRealtimeSessionProtocol({}),
)
assert allowed is True
assert promoted is None
assert native.input_since_commit is True
assert handler._assistant_playback_active(session) is True
next_speech = _native_audio_payload(samples=16_000)
assert "new_user_turn" not in next_speech
@pytest.mark.asyncio
async def test_realtime_committed_overlap_promotes_after_response_done():
handler, session = _auto_response_context("sid-auto-realtime-committed-input", playback_active=True)
session.capabilities = minicpmo45_native_capabilities()
response_id = session.active_response_id
assert response_id is not None
native = handler._runtime_session_state(session)
native.committed_audio_payload = _native_audio_payload()
native.deferred_response_create = True
session.accumulate_overlap_speech(1000)
session.end_response(commit_text=False, preserve_request=True)
allowed, promoted = await handler._apply_outbound_session_event(
{
"type": "response.done",
"response_id": response_id,
"epoch": session.epoch,
"status": "completed",
},
session=session,
actor=DuplexWebSocketActor(TimedWebSocket()),
native=native,
realtime_protocol=NativeRealtimeSessionProtocol({}),
)
assert allowed is True
assert promoted is not None
assert promoted["force_listen"] is False
assert native.committed_audio_payload is promoted
assert native.deferred_response_create is False
assert "new_user_turn" not in promoted
@pytest.mark.asyncio
async def test_auto_response_committed_overlap_does_not_precreate_empty_response():
session_id = "sid-auto-overlap-no-empty-response"
request_id = f"duplex-{session_id}-e0-stage0"
def _stage_output(samples: int, *, turn_end: bool = False):
return SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[
SimpleNamespace(
text="hello",
multimodal_output={
"audio": np.zeros(samples, dtype=np.float32),
"sr": 24000,
"meta": {
"turn_end": turn_end,
"duplex_turn_id": 0,
"duplex_epoch": 0,
},
},
)
],
)
first_append = {
"operation": "append",
"session_id": session_id,
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {
"supported": True,
"implementation_level": "model_native_duplex",
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
},
}
],
"data_plane_outputs": [_stage_output(2400)],
}
promoted_append = {
"operation": "append",
"session_id": session_id,
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [],
"data_plane_outputs": [],
}
engine = FakeEngineClient(
append_results=[first_append, promoted_append],
collect_outputs=[[_stage_output(4800, turn_end=True)]],
collect_delay_s=0.1,
)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=2,
)
overlap_sent = False
def commit_overlap_after_first_response(ws: TimedWebSocket, data: dict[str, Any]) -> None:
nonlocal overlap_sent
if data.get("type") != "response.audio.delta" or overlap_sent:
return
overlap_sent = True
chunk = {
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(3200, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"duration_ms": 200,
"is_speech": True,
}
for _ in range(4):
ws.put(chunk)
ws.put({"type": "input_audio_buffer.commit", "final": True})
ws = TimedWebSocket(on_send=commit_overlap_after_first_response, receive_timeout_s=3)
create = _native_realtime_session_update(session_id)
create["session"]["extra_body"]["auto_response"] = True
ws.put(create)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"duration_ms": 1000,
"is_speech": True,
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
handler_task = asyncio.create_task(handler.handle_realtime_session(ws))
try:
for _ in range(300):
if len(engine.appended) >= 2:
break
await asyncio.sleep(0.01)
else:
pytest.fail(
f"committed overlap was not promoted after response.done: "
f"appends={len(engine.appended)}, events={ws.sent_types()}"
)
assert overlap_sent is True
assert len(engine.appended) == 2
assert engine.appended[1][3] is True
assert ws.sent_types().count("response.created") == 1
finally:
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=3)
assert ws.sent_types().count("response.done") >= 1
def test_auto_response_barge_in_does_not_mark_serving_new_user_turn_payload():
handler, session = _auto_response_context("sid-auto-barge-new-turn", playback_active=True)
payload = _native_audio_payload()
assert handler._should_force_listen_for_auto_response_overlap(session, {"force_barge_in": True}, payload) is False
assert "new_user_turn" not in payload
assert "new_user_turn_prefix_variant" not in payload
assert session.turn_id == 0
assert payload.get("force_speak") is not True
def test_native_realtime_barge_in_payload_does_not_create_serving_turn_boundary():
payload: dict[str, object] = {
"type": "audio",
"audio": _pcm_f32_b64(16000, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"is_speech": True,
}
session = DuplexSession(session_id="sid-realtime-barge", config=DuplexSessionConfig())
session.turn_id = 2
assert "new_user_turn" not in payload
assert "new_user_turn_prefix_variant" not in payload
assert session.turn_id == 2
def test_auto_response_force_barge_in_has_no_waiting_turn_variant():
handler, session = _auto_response_context("sid-auto-barge-keeps-variant")
payload = _native_audio_payload()
assert (
handler._should_force_listen_for_auto_response_overlap(
session,
{"force_barge_in": True},
payload,
)
is False
)
def test_duplex_endpoint_requires_explicit_session_mode_duplex():
assert should_enable_duplex_endpoint(None) is False
assert should_enable_duplex_endpoint([]) is False
assert should_enable_duplex_endpoint([SimpleNamespace(session_mode="turn")]) is False
assert should_enable_duplex_endpoint([{"session_mode": "turn"}]) is False
assert (
should_enable_duplex_endpoint(
[
SimpleNamespace(session_mode="turn"),
SimpleNamespace(session_mode="duplex"),
]
)
is True
)
assert should_enable_duplex_endpoint([{"session_mode": "duplex"}]) is True
def test_duplex_endpoint_supports_top_level_session_mode(tmp_path):
config_path = tmp_path / "stage_config.yaml"
config_path.write_text(
"""
session_mode: duplex
stage_args:
- stage_id: 0
engine_args: {}
""",
encoding="utf-8",
)
assert should_enable_duplex_endpoint([], config_path=str(config_path)) is True
def test_duplex_handler_splits_data_plane_audio_list_into_deltas():
import torch
data_plane = _test_data_plane()
output = SimpleNamespace(
finished=True,
outputs=[
SimpleNamespace(
text="hello",
multimodal_output={
"audio": [
torch.zeros(10, dtype=torch.float32),
torch.zeros(20, dtype=torch.float32),
],
"sr": 24000,
},
)
],
)
native_results = list(data_plane.project_output(output))
assert [result["audio_data"] for result in native_results] == ["wav-10", "wav-20"]
assert [result["text"] for result in native_results] == ["hello", ""]
assert [result["end_of_turn"] for result in native_results] == [False, True]
def test_duplex_listen_latent_does_not_poison_cumulative_audio_offset():
import torch
data_plane = _test_data_plane()
request_id = "duplex-duplex-sess-stage0"
# A model-listen decision wraps the segment with a latent tensor that is
# NOT reply audio; it must not advance the cumulative audio offset.
listen_output = SimpleNamespace(
request_id=request_id,
finished=True,
outputs=[],
multimodal_output={
"duplex_native_decision": "listen",
"model_listen": True,
"latent": torch.zeros(331776, dtype=torch.float32),
"meta": {"sr": 24000},
},
)
listen_results = list(data_plane.project_output(listen_output))
assert [result.get("is_listen") for result in listen_results] == [True]
# The first speak unit carries cumulative stage-1 audio far smaller than
# the listen latent; it must still be delivered from sample 0.
speak_output = SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[
SimpleNamespace(
text=" It was a very",
multimodal_output={},
)
],
multimodal_output={
"audio": torch.zeros(32768, dtype=torch.float32),
"sr": 24000,
},
)
speak_results = list(data_plane.project_output(speak_output))
assert [result.get("audio_data") for result in speak_results] == ["wav-32768"]
assert speak_results[0]["text"] == " It was a very"
def test_direct_listen_decision_survives_inner_completion_metadata():
inner_output = SimpleNamespace(
outputs=[
SimpleNamespace(
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
)
]
)
decision = MiniCPMO45DuplexRuntimeExtension().decide_output(
stage_id=0,
final_stage_id=1,
segment_finished=True,
segment_token_ids=(151705,),
segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}},
output=inner_output,
)
assert decision is not None
output = attach_duplex_output_decision(
OmniRequestOutput.from_stage_output(
inner_output,
request_id="duplex-direct-listen",
finished=True,
stage_id=0,
final_output_type=decision.final_output_type,
metrics={
"stage_metrics": {
"0": {
"vllm_ttft_ms": 125.0,
"vllm_tpot_ms": 18.0,
"vllm_itl_ms": 17.5,
"vllm_itls_ms": [17.0, 18.0],
}
}
},
),
decision,
)
results = list(_test_data_plane().project_output(output))
assert len(results) == 1
assert results[0]["is_listen"] is True
assert results[0]["model_listen"] is True
assert results[0]["listen_source"] == "model_listen"
assert results[0]["stage_metrics"] == output.metrics["stage_metrics"]
def test_duplex_segment_text_is_attached_once_across_streaming_batches():
import torch
class _ChatService:
model_config = _ModelConfig()
def create_audio(self, audio_obj):
return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}")
data_plane = _test_data_plane()
request_id = "duplex-duplex-sess-stage0"
session = DuplexSession(
session_id="sid-auto-respond",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
def _speak(total_samples: int, text: str, *, finished: bool):
output = SimpleNamespace(
request_id=request_id,
finished=finished,
outputs=[SimpleNamespace(text=text, multimodal_output={})],
multimodal_output={
"audio": torch.zeros(total_samples, dtype=torch.float32),
"sr": 24000,
},
)
return _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
def _audio(results):
return [r for r in results if r.get("is_listen") is False]
def _listen(results):
return [r for r in results if r.get("is_listen") is True]
# One talker segment streams several cumulative-audio batches, each
# carrying the SAME segment text; only the first may attach it.
assert [r["text"] for r in _audio(_speak(100, " movie called", finished=False))] == [" movie called"]
assert [r["text"] for r in _audio(_speak(220, " movie called", finished=False))] == [""]
finished_results = _speak(300, " movie called", finished=True)
assert [r["text"] for r in _audio(finished_results)] == [""]
assert _listen(finished_results) == []
# Continuation units re-run the talker with the SAME handed text past a
# finished boundary (every engine segment ends finished=True); the text
# must stay suppressed or each continuation duplicates the transcript.
assert [r["text"] for r in _audio(_speak(400, " movie called", finished=False))] == [""]
assert [r["text"] for r in _audio(_speak(450, " movie called", finished=True))] == [""]
# A genuinely new segment text is attached in full.
assert [r["text"] for r in _audio(_speak(500, " Titanic", finished=True))] == [" Titanic"]
# Text growing within a segment is delivered as suffix deltas.
assert [r["text"] for r in _audio(_speak(600, " by", finished=False))] == [" by"]
assert [r["text"] for r in _audio(_speak(700, " by James", finished=True))] == [" James"]
# A segment whose finished batch slices to an EMPTY audio delta (all
# samples already delivered) must not block the next segment's text.
assert [r["text"] for r in _audio(_speak(800, " James Cameron. The", finished=False))] == [" James Cameron. The"]
assert _speak(800, " James Cameron. The", finished=True) == []
assert [r["text"] for r in _audio(_speak(900, " 997.", finished=False))] == [" 997."]
def test_duplex_data_plane_output_prefers_audio_segment_text_metadata():
data_plane = _test_data_plane()
request_id = "duplex-sid-native-segment-text-e0-stage0"
segment_text = "你好,有什么想聊的吗?"
cumulative_text = segment_text * 2
output = SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[SimpleNamespace(text=cumulative_text, multimodal_output={})],
multimodal_output={
"audio": np.zeros(24000, dtype=np.float32),
"sr": 24000,
"meta.llm_output_text_utf8": np.frombuffer(segment_text.encode("utf-8"), dtype=np.uint8),
},
)
results = list(data_plane.project_output(output))
assert len(results) == 1
assert results[0]["audio_data"] == "wav-24000"
assert results[0]["text"] == segment_text
next_output_without_segment_metadata = SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[SimpleNamespace(text=segment_text * 3, multimodal_output={})],
multimodal_output={
"audio": np.zeros(30720, dtype=np.float32),
"sr": 24000,
},
)
next_results = list(data_plane.project_output(next_output_without_segment_metadata))
assert len(next_results) == 1
assert next_results[0]["audio_data"] == "wav-6720"
assert next_results[0]["text"] == ""
@pytest.mark.parametrize(
("request_id", "second_samples", "expected_second_audio"),
[
("duplex-sid-model-turn-audio-e0-stage0", 36000, "wav-12000"),
("duplex-sid-model-turn-audio-restart-e0-stage0", 12000, "wav-12000"),
],
ids=["cumulative-across-model-turns", "upstream-cumulative-audio-restarts"],
)
def test_duplex_audio_cursor_across_model_turns(
request_id: str,
second_samples: int,
expected_second_audio: str,
):
data_plane = _test_data_plane()
def output(turn_id: int, samples: int):
return SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[SimpleNamespace(text="", multimodal_output={})],
multimodal_output={
"audio": np.zeros(samples, dtype=np.float32),
"sr": 24000,
"meta.duplex_turn_id": np.array([turn_id], dtype=np.int32),
},
)
first = list(data_plane.project_output(output(0, 24000)))
second = list(data_plane.project_output(output(1, second_samples)))
assert first[0]["audio_data"] == "wav-24000"
assert second[0]["audio_data"] == expected_second_audio
def test_duplex_data_plane_accepts_model_turn_output_across_client_commits():
data_plane = _test_data_plane()
session = DuplexSession(
session_id="sid-auto-stale-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.commit_native_audio_input()
session.commit_native_audio_input()
session.turn_id = 1
session.epoch = 0
request_id = "duplex-sid-auto-stale-turn-stage0"
output = SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[SimpleNamespace(text="old tail", multimodal_output={})],
multimodal_output={
"audio": np.zeros(24000, dtype=np.float32),
"sr": 24000,
"meta.duplex_turn_id": np.array([1], dtype=np.int32),
"meta.llm_output_text_utf8": np.frombuffer("旧尾巴".encode(), dtype=np.uint8),
},
)
results = list(data_plane.project_output(output, context=_data_plane_context(session)))
assert len(results) == 1
assert results[0]["audio_data"] == "wav-24000"
assert results[0]["model_turn_id"] == 1
def test_duplex_data_plane_accepts_active_response_turn_identity():
data_plane = _test_data_plane()
session = DuplexSession(
session_id="sid-active-response-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.turn_id = 1
session.epoch = 0
session.begin_response(turn_id=0)
output = SimpleNamespace(
request_id="duplex-sid-active-response-turn-stage0",
finished=True,
outputs=[SimpleNamespace(text="", multimodal_output={})],
multimodal_output={
"audio": np.zeros(10, dtype=np.float32),
"sr": 24000,
"meta.duplex_turn_id": np.array([0], dtype=np.int32),
"meta.duplex_epoch": np.array([0], dtype=np.int32),
},
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert len(results) == 1
def test_duplex_data_plane_listen_preserves_model_turn_identity():
data_plane = _test_data_plane()
session = DuplexSession(
session_id="sid-listen-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
output = SimpleNamespace(
request_id="duplex-sid-listen-turn-stage0",
finished=True,
outputs=[SimpleNamespace(text="", token_ids=[], multimodal_output={})],
multimodal_output={
"duplex_native_decision": "listen",
"meta.duplex_turn_id": np.array([2], dtype=np.int32),
"meta.duplex_epoch": np.array([0], dtype=np.int32),
},
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert len(results) == 1
assert results[0]["is_listen"] is True
assert results[0]["model_turn_id"] == 2
def test_duplex_data_plane_does_not_drop_future_model_turn_while_response_active():
data_plane = _test_data_plane()
session = DuplexSession(
session_id="sid-future-response-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.turn_id = 0
session.epoch = 0
session.begin_response(turn_id=0)
output = SimpleNamespace(
request_id="duplex-sid-future-response-turn-stage0",
finished=False,
outputs=[SimpleNamespace(text="", multimodal_output={})],
multimodal_output={
"audio": np.zeros(10, dtype=np.float32),
"sr": 24000,
"meta.duplex_turn_id": np.array([1], dtype=np.int32),
"meta.duplex_epoch": np.array([0], dtype=np.int32),
},
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert len(results) == 1
assert results[0]["model_turn_id"] == 1
assert results[0]["audio_data"] == "wav-10"
def _duplex_tts_output(
*,
request_id: str,
samples: int,
finished: bool,
text: str = "hello",
tts_is_last_chunk: bool = False,
turn_end: bool = False,
turn_id: int = 0,
token_ids: list[int] | None = None,
):
return SimpleNamespace(
request_id=request_id,
finished=finished,
outputs=[SimpleNamespace(text=text, token_ids=list(token_ids or []), multimodal_output={})],
multimodal_output={
"audio": np.zeros(samples, dtype=np.float32),
"sr": 24000,
"meta.tts_is_last_chunk": np.array([int(tts_is_last_chunk)], dtype=np.int32),
"meta.turn_end": np.array([int(turn_end)], dtype=np.int32),
"meta.duplex_turn_id": np.array([turn_id], dtype=np.int32),
"meta.duplex_epoch": np.array([0], dtype=np.int32),
},
)
@pytest.mark.parametrize(
(
"finished",
"tts_is_last_chunk",
"turn_end",
"consume_audio_first",
"expected_end_of_turn",
"expected_audio",
),
[
(False, True, False, False, False, "wav-24000"),
(True, True, False, True, False, ""),
(True, True, True, True, True, ""),
],
ids=[
"last-audio-batch-does-not-end-response",
"scheduler-eos-only-ends-tts-segment",
"explicit-turn-end-metadata-ends-response",
],
)
def test_duplex_auto_response_tts_boundary(
finished: bool,
tts_is_last_chunk: bool,
turn_end: bool,
consume_audio_first: bool,
expected_end_of_turn: bool,
expected_audio: str,
):
data_plane = _test_data_plane()
request_id = "duplex-sid-tts-boundary-e0-stage0"
session = DuplexSession(
session_id="sid-tts-boundary",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.bind_response_turn(0)
if consume_audio_first:
data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32))
output = _duplex_tts_output(
request_id=request_id,
samples=24000,
finished=finished,
tts_is_last_chunk=tts_is_last_chunk,
turn_end=turn_end,
token_ids=[151645] if finished else None,
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert len(results) == 1
boundary = results[0]
assert boundary["stage_role"] == "tts"
assert boundary["end_of_turn"] is expected_end_of_turn
assert boundary["abort_data_plane_request"] is True
assert boundary["audio_data"] == expected_audio
def test_duplex_auto_response_tts_scheduler_eos_fallback():
data_plane = _test_data_plane()
request_id = "duplex-sid-tts-eos-no-profile-e0-stage0"
session = DuplexSession(
session_id="sid-tts-eos-no-profile",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.bind_response_turn(0)
data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32))
output = _duplex_tts_output(
request_id=request_id,
samples=24000,
finished=True,
tts_is_last_chunk=False,
token_ids=[151645],
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert len(results) == 1
assert results[0]["stage_role"] == "tts"
assert results[0]["abort_data_plane_request"] is True
def test_duplex_auto_response_rearms_tts_boundary_for_next_segment_in_same_turn():
data_plane = _test_data_plane()
request_id = "duplex-sid-next-segment-same-turn-e0-stage0"
session = DuplexSession(
session_id="sid-next-segment-same-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.bind_response_turn(0)
data_plane.begin_request(request_id)
first_segment = _duplex_tts_output(
request_id=request_id,
samples=24000,
finished=False,
tts_is_last_chunk=True,
turn_id=0,
)
first_results = _project_data_plane(
data_plane,
{"data_plane_outputs": [first_segment]},
session=session,
)
assert len(first_results) == 1
assert first_results[0]["abort_data_plane_request"] is True
data_plane.begin_request(request_id)
second_segment = _duplex_tts_output(
request_id=request_id,
samples=48000,
finished=False,
text="hello again",
tts_is_last_chunk=True,
turn_id=0,
)
second_results = _project_data_plane(
data_plane,
{"data_plane_outputs": [second_segment]},
session=session,
)
assert len(second_results) == 1
assert second_results[0]["audio_data"] == "wav-24000"
assert second_results[0]["abort_data_plane_request"] is True
def test_duplex_turn_end_is_not_swallowed_by_finished_segment_fallback():
data_plane = _test_data_plane()
request_id = "duplex-sid-turn-end-after-segment-e0-stage0"
session = DuplexSession(
session_id="sid-turn-end-after-segment",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.begin_response(turn_id=0)
session.bind_request(request_id)
_project_data_plane(
data_plane,
{
"data_plane_outputs": [
_duplex_tts_output(
request_id=request_id,
samples=24000,
finished=False,
tts_is_last_chunk=True,
)
]
},
session=session,
)
output = _duplex_tts_output(
request_id=request_id,
samples=24000,
finished=True,
text="",
tts_is_last_chunk=False,
turn_end=True,
token_ids=[151645],
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert len(results) == 1
assert results[0]["stage_role"] == "tts"
assert results[0]["end_of_turn"] is True
duplicate = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert not any(result.get("end_of_turn") is True for result in duplicate)
def test_duplex_auto_response_discards_terminal_only_audio_before_response_creation():
data_plane = _test_data_plane()
request_id = "duplex-sid-terminal-only-audio-e0-stage0"
session = DuplexSession(
session_id="sid-terminal-only-audio",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.bind_request(request_id)
first_audio = _duplex_tts_output(
request_id=request_id,
samples=3840,
finished=False,
text="",
)
assert _project_data_plane(data_plane, {"data_plane_outputs": [first_audio]}, session=session) == []
terminal = _duplex_tts_output(
request_id=request_id,
samples=3840,
finished=True,
text="",
tts_is_last_chunk=True,
turn_end=True,
token_ids=[151645],
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [terminal]}, session=session)
assert len(results) == 1
assert results[0]["audio_data"] == ""
assert results[0]["end_of_turn"] is True
assert session.active_request_id == request_id
assert not data_plane.is_terminal(request_id)
def test_duplex_auto_response_releases_buffered_audio_when_transcript_arrives():
data_plane = _test_data_plane()
request_id = "duplex-sid-delayed-transcript-e0-stage0"
session = DuplexSession(
session_id="sid-delayed-transcript",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
audio_before_text = _duplex_tts_output(
request_id=request_id,
samples=100,
finished=False,
text="",
)
assert _project_data_plane(data_plane, {"data_plane_outputs": [audio_before_text]}, session=session) == []
audio_with_text = _duplex_tts_output(
request_id=request_id,
samples=200,
finished=False,
text="hello",
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [audio_with_text]}, session=session)
assert [result["audio_data"] for result in results] == ["wav-100", "wav-100"]
assert [result["text"] for result in results] == ["", "hello"]
def test_duplex_auto_response_releases_buffered_audio_on_text_only_terminal():
data_plane = _test_data_plane()
request_id = "duplex-sid-delayed-terminal-text-e0-stage0"
session = DuplexSession(
session_id="sid-delayed-terminal-text",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
audio_before_text = _duplex_tts_output(
request_id=request_id,
samples=100,
finished=False,
text="",
turn_id=1,
)
assert _project_data_plane(data_plane, {"data_plane_outputs": [audio_before_text]}, session=session) == []
textless_turn_terminal = _duplex_tts_output(
request_id=request_id,
samples=100,
finished=True,
text="",
tts_is_last_chunk=True,
turn_end=True,
turn_id=1,
token_ids=[151645],
)
terminal_results = list(
data_plane.project(
{"data_plane_outputs": [textless_turn_terminal]},
context=_data_plane_context(session),
)
)
assert len(terminal_results) == 1
assert terminal_results[0]["audio_data"] == ""
assert terminal_results[0]["end_of_turn"] is True
text_only_terminal = SimpleNamespace(
request_id=request_id,
finished=True,
outputs=[SimpleNamespace(text="hello", token_ids=[151645], multimodal_output={})],
multimodal_output={
"sr": 24000,
"meta.tts_is_last_chunk": np.array([1], dtype=np.int32),
"meta.turn_end": np.array([1], dtype=np.int32),
"meta.duplex_turn_id": np.array([2], dtype=np.int32),
"meta.duplex_epoch": np.array([0], dtype=np.int32),
},
)
results = _project_data_plane(data_plane, {"data_plane_outputs": [text_only_terminal]}, session=session)
assert len(results) == 1
assert results[0]["audio_data"] == "wav-100"
assert results[0]["text"] == "hello"
assert results[0]["end_of_turn"] is True
assert results[0]["abort_data_plane_request"] is True
assert not data_plane.has_pending_audio(request_id)
@pytest.mark.asyncio
async def test_native_append_propagates_current_turn_fence_to_engine():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
session = DuplexSession(
session_id="sid-fenced-append",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.turn_id = 2
session.begin_response(turn_id=2)
ws = TimedWebSocket()
await handler._append_runtime_input(
session,
{"duplex_turn_id": 2, "audio": "", "format": "pcm_f32le"},
final=True,
send_json=ws.send_json,
mode="append_audio_chunk",
expected_epoch=0,
)
assert engine.appended_fences == [DuplexFence("sid-fenced-append", epoch=0, turn_id=2)]
@pytest.mark.asyncio
async def test_minicpmo_auto_response_tts_segment_boundary_appends_silence_unit():
request_id = "duplex-sid-segment-boundary-e0-stage0"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-segment-boundary",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
session.mark_audio_sent(100)
_install_direct_silence_scheduler(handler, session)
ws = TimedWebSocket()
await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"text": "",
"audio_data": "",
"audio_format": "pcm16",
"end_of_turn": False,
"abort_data_plane_request": True,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
await asyncio.sleep(0.01)
assert len(engine.appended) == 1
_, mode, payload, final = engine.appended[0]
assert mode == "append_audio_chunk"
assert payload["format"] == "pcm_f32le"
assert payload["duplex_turn_id"] == 0
assert "force_speak" not in payload
assert final is False
assert session.active_response_id is not None
assert "response.done" not in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_auto_response_pre_response_tts_boundary_continues_model_turn():
request_id = "duplex-sid-pre-response-boundary-e0-stage0"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
session = DuplexSession(
session_id="sid-pre-response-boundary",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
_install_direct_silence_scheduler(handler, session)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"model_turn_id": session.turn_id,
"text": "",
"audio_data": "",
"audio_format": "pcm16",
"end_of_turn": False,
"abort_data_plane_request": True,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
await asyncio.sleep(0.01)
assert close_reason is None
assert emitted is False
assert session.active_response_id is None
assert ws.sent == []
assert len(engine.appended) == 1
_, mode, payload, final = engine.appended[0]
assert mode == "append_audio_chunk"
assert payload["duplex_turn_id"] == session.turn_id
assert final is False
@pytest.mark.asyncio
async def test_minicpmo_pre_response_continuation_drops_after_model_turn_ends():
request_id = "duplex-sid-stale-pre-response-boundary-e0-stage0"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
session = DuplexSession(
session_id="sid-stale-pre-response-boundary",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
model_turn_id = session.turn_id
native = handler._runtime_session_state(session)
async def _stale_before_append(payload: object, **kwargs: Any) -> bool:
session.complete_model_turn(model_turn_id)
if handler._native_silence_continuation_is_stale(
session,
request_id=kwargs["request_id"],
response_id=kwargs["response_id"],
response_owned=kwargs["response_owned"],
expected_epoch=kwargs["expected_epoch"],
expected_incarnation=kwargs["expected_incarnation"],
expected_model_turn_id=kwargs["expected_model_turn_id"],
):
return False
append_ok, _ = await handler._append_runtime_input(
session,
payload,
final=False,
send_json=kwargs["send_json"],
mode="append_audio_chunk",
expected_epoch=kwargs["expected_epoch"],
)
return append_ok
native.silence_continuation_scheduler = _stale_before_append
await handler._maybe_continue_native_response(
TimedWebSocket().send_json,
session=session,
expected_epoch=session.epoch,
expected_model_turn_id=model_turn_id,
)
await asyncio.sleep(0.01)
assert engine.appended == []
@pytest.mark.asyncio
async def test_minicpmo_auto_response_continuation_stops_at_large_safety_boundary():
request_id = "duplex-sid-long-response-e0-stage0"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
session = DuplexSession(
session_id="sid-long-response",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
session.mark_audio_sent(100)
_install_direct_silence_scheduler(handler, session)
ws = TimedWebSocket()
for _ in range(handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS):
await handler._maybe_continue_native_response(
ws.send_json,
session=session,
expected_epoch=session.epoch,
)
await asyncio.sleep(0.02)
response_id = session.active_response_id
assert response_id is not None
assert len(engine.appended) == handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
assert all(payload["duplex_turn_id"] == 0 for _, _, payload, _ in engine.appended)
assert all(not {"force_speak", "force_listen"} & payload.keys() for _, _, payload, _ in engine.appended[:-1])
assert engine.appended[-1][2]["force_listen"] is True
assert handler._native_response_continuations_remaining(session, response_id) is False
await handler._maybe_continue_native_response(
ws.send_json,
session=session,
expected_epoch=session.epoch,
)
assert len(engine.appended) == handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
assert ws.sent_types()[-2:] == ["response.listen", "response.done"]
assert session.active_response_id is None
assert session.active_request_id == request_id
assert session.turn_id == 1
@pytest.mark.asyncio
async def test_minicpmo_auto_response_boundary_listen_closes_response():
request_id = "duplex-sid-boundary-listen-e0-stage0"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
session = DuplexSession(
session_id="sid-boundary-listen",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response(turn_id=0)
session.bind_request(request_id)
native = handler._runtime_session_state(session)
native.continuation_owner_id = f"response:{response_id}"
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
_install_direct_silence_scheduler(handler, session)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "llm",
"is_listen": True,
"model_listen": True,
"data_plane_request_id": request_id,
"end_of_turn": False,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is True
assert engine.appended == []
assert ws.sent_types() == ["response.listen", "response.done"]
# The listen terminates the precreated response: clients demultiplex the
# decision by its id, so the payload must carry the response identity.
listen_payload, done_payload = ws.sent[-2], ws.sent[-1]
assert listen_payload["response_id"] == response_id
assert done_payload["response_id"] == response_id
assert session.active_response_id is None
assert session.active_request_id == request_id
assert session.turn_id == 1
assert native.continuation_owner_id is None
@pytest.mark.asyncio
async def test_minicpmo_auto_response_boundary_allows_next_model_turn_response():
request_id = "duplex-sid-boundary-next-turn-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-boundary-next-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.bind_request(request_id)
native = handler._runtime_session_state(session)
native.continuation_owner_id = f"response:{session.active_response_id}"
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
ws = TimedWebSocket()
await handler._maybe_continue_native_response(
ws.send_json,
session=session,
expected_epoch=session.epoch,
)
assert session.active_response_id is None
assert session.turn_id == 1
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"text": "next response",
"audio_data": "next-audio",
"audio_format": "pcm16",
"audio_duration_ms": 100,
"end_of_turn": False,
"model_turn_id": 1,
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is True
assert ws.sent_types() == [
"response.listen",
"response.done",
"response.created",
"response.speak",
"response.output_audio.delta",
]
assert session.active_response_turn_id == 1
@pytest.mark.asyncio
async def test_minicpmo_auto_response_boundary_does_not_close_new_epoch_response():
request_id = "duplex-sid-boundary-race-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-boundary-race",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
old_response_id = session.begin_response(turn_id=0)
session.bind_request(request_id)
native = handler._runtime_session_state(session)
native.continuation_owner_id = f"response:{old_response_id}"
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
sent: list[dict[str, Any]] = []
new_response_ids: list[str] = []
async def _barge_in_on_listen(payload: dict[str, Any]) -> None:
sent.append(payload)
if payload["type"] == "response.listen":
session.barge_in()
session.bind_request("duplex-sid-boundary-race-e1-stage0")
new_response_ids.append(session.begin_response(turn_id=1))
await handler._maybe_continue_native_response(
_barge_in_on_listen,
session=session,
expected_epoch=session.epoch,
)
assert [payload["type"] for payload in sent] == ["response.listen"]
assert len(new_response_ids) == 1
assert session.active_response_id == new_response_ids[0]
@pytest.mark.asyncio
async def test_minicpmo_auto_response_pre_speak_listen_continues_same_response():
request_id = "duplex-sid-pre-speak-listen-e0-stage0"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
session = DuplexSession(
session_id="sid-pre-speak-listen",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
_install_direct_silence_scheduler(handler, session)
ws = TimedWebSocket()
await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "llm",
"is_listen": True,
"model_listen": True,
"data_plane_request_id": request_id,
"end_of_turn": False,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
await asyncio.sleep(0.01)
assert len(engine.appended) == 1
assert session.active_response_id == response_id
assert session.active_request_id == request_id
assert "response.listen" not in ws.sent_types()
assert "response.done" not in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_request():
request_id = "duplex-sid-listen-before-response-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-listen-before-response",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "llm",
"is_listen": True,
"model_listen": True,
"data_plane_request_id": request_id,
"end_of_turn": False,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is True
assert session.active_request_id == request_id
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
assert ws.sent_types().count("response.listen") == 1
@pytest.mark.asyncio
async def test_minicpmo_auto_response_listen_without_response_does_not_defer_commit():
session_id = "sid-listen-before-response-commit"
request_id = duplex_resource_request_id(DuplexFence(session_id), "stage0")
inner_output = SimpleNamespace(
outputs=[
SimpleNamespace(
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
)
]
)
decision = MiniCPMO45DuplexRuntimeExtension().decide_output(
stage_id=0,
final_stage_id=1,
segment_finished=True,
segment_token_ids=(151705,),
segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}},
output=inner_output,
)
assert decision is not None
listen_output = attach_duplex_output_decision(
OmniRequestOutput.from_stage_output(
inner_output,
request_id=request_id,
finished=True,
stage_id=0,
final_output_type=decision.final_output_type,
),
decision,
)
append_result = {
"operation": "append",
"session_id": session_id,
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {
"supported": True,
"implementation_level": "model_native_duplex",
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
},
}
],
"data_plane_outputs": [listen_output],
}
engine = FakeEngineClient(append_result=append_result)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
listen_count = 0
def append_residual_after_listen(ws: TimedWebSocket, data: dict[str, Any]) -> None:
nonlocal listen_count
if data.get("type") == "response.listen":
listen_count += 1
if listen_count == 1:
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(3_200),
"format": "pcm_f32le",
"sample_rate_hz": 16_000,
"duration_ms": 200,
"is_speech": True,
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
elif data.get("type") == "input_audio_buffer.committed":
asyncio.get_running_loop().call_later(0.05, ws.put, {"type": "session.close"})
ws = TimedWebSocket(on_send=append_residual_after_listen, receive_timeout_s=2)
create = _native_realtime_session_update(session_id)
create["session"]["extra_body"]["auto_response"] = True
ws.put(create)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16_000),
"format": "pcm_f32le",
"sample_rate_hz": 16_000,
"duration_ms": 1_000,
"is_speech": True,
}
)
await handler.handle_realtime_session(ws)
committed = [event for event in ws.sent if event.get("type") == "input_audio_buffer.committed"]
assert len(engine.appended) == 2
assert engine.appended[1][3] is True
assert len(committed) == 1
assert committed[0].get("event", {}).get("overlap_deferred") is not True
assert committed[0].get("event", {}).get("response_create_deferred") is not True
assert "response.created" not in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_pcm_f32_residual_commit_releases_pending_input_bytes():
session_id = "sid-pcm-residual-byte-accounting"
control_result = {
"operation": "append",
"session_id": session_id,
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [],
}
engine = FakeEngineClient(control_result=control_result)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket(receive_timeout_s=2)
create = _native_realtime_session_update(session_id)
create["session"]["extra_body"]["auto_response"] = True
ws.put(create)
handler_task = asyncio.create_task(handler.handle_realtime_session(ws))
try:
for _ in range(100):
session = handler._registry.get(session_id)
if session is not None and "session.created" in ws.sent_types():
break
await asyncio.sleep(0.01)
else:
pytest.fail("native duplex session did not open")
pending_bytes_after_turn: list[int] = []
for turn_index in range(3):
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(19_200),
"format": "pcm_f32le",
"sample_rate_hz": 16_000,
"duration_ms": 1_200,
"is_speech": True,
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
expected_appends = (turn_index + 1) * 2
for _ in range(100):
if len(engine.appended) >= expected_appends:
await asyncio.sleep(0)
break
await asyncio.sleep(0.01)
else:
pytest.fail(f"turn {turn_index} residual append did not complete")
pending_bytes_after_turn.append(session.pending_input_bytes)
finally:
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert pending_bytes_after_turn == [0, 0, 0]
assert not any(
event.get("type") == "error"
and isinstance(event.get("error"), dict)
and event["error"].get("code") == "input_backpressure"
for event in ws.sent
)
@pytest.mark.asyncio
async def test_minicpmo_auto_response_post_speak_listen_continues_same_response():
request_id = "duplex-sid-post-speak-listen-e0-stage0"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
session = DuplexSession(
session_id="sid-post-speak-listen",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
session.mark_audio_sent(100)
_install_direct_silence_scheduler(handler, session)
ws = TimedWebSocket()
await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "llm",
"is_listen": True,
"model_listen": True,
"data_plane_request_id": request_id,
"end_of_turn": False,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
await asyncio.sleep(0.01)
assert len(engine.appended) == 1
assert session.active_response_id == response_id
assert session.active_request_id == request_id
assert "response.listen" not in ws.sent_types()
assert "response.done" not in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_auto_response_turn_end_preserves_resumable_request():
request_id = "duplex-sid-turn-end-preserve-request-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-turn-end-preserve-request",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.bind_request(request_id)
ws = TimedWebSocket()
await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"text": "done",
"audio_data": "audio",
"audio_format": "pcm16",
"audio_duration_ms": 100,
"end_of_turn": True,
"model_turn_id": 0,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert session.active_response_id is None
assert session.active_request_id == request_id
assert "response.done" in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_auto_response_empty_turn_end_emits_model_listen():
request_id = "duplex-sid-empty-turn-end-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-empty-turn-end",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"text": "",
"audio_data": "",
"audio_format": "pcm16",
"end_of_turn": True,
"abort_data_plane_request": True,
"model_turn_id": 0,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is True
assert session.turn_id == 1
assert session.active_request_id == request_id
assert session.active_response_id is None
assert ws.sent_types() == ["response.listen"]
assert ws.sent[0]["model_listen"] is True
assert ws.sent[0]["reason"] == "model_turn_completed_without_output"
@pytest.mark.asyncio
async def test_minicpmo_auto_response_drops_late_audio_from_completed_model_turn():
request_id = "duplex-sid-late-completed-turn-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-late-completed-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
session.complete_model_turn(0)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"text": "late duplicate",
"audio_data": "late-audio",
"audio_format": "pcm16",
"audio_duration_ms": 100,
"end_of_turn": True,
"model_turn_id": 0,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is False
assert session.turn_id == 1
assert session.active_response_id is None
assert "response.created" not in ws.sent_types()
assert "response.done" not in ws.sent_types()
def test_duplex_data_plane_drops_stale_epoch_audio_after_barge_in():
class _ChatService:
model_config = _ModelConfig()
def create_audio(self, audio_obj):
return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}")
data_plane = _test_data_plane()
session = DuplexSession(
session_id="sid-auto-stale-epoch",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.turn_id = 1
session.epoch = 2
output = SimpleNamespace(
request_id="duplex-sid-auto-stale-epoch-stage0",
finished=False,
outputs=[SimpleNamespace(text="old epoch", multimodal_output={})],
multimodal_output={
"audio": np.zeros(24000, dtype=np.float32),
"sr": 24000,
"meta.duplex_epoch": np.array([1], dtype=np.int32),
"meta.duplex_turn_id": np.array([1], dtype=np.int32),
"meta.llm_output_text_utf8": np.frombuffer("旧轮".encode(), dtype=np.uint8),
},
)
assert list(data_plane.project_output(output, context=_data_plane_context(session))) == []
def test_duplex_data_plane_text_delta_preserves_repeated_suffix_growth():
data_plane = _test_data_plane()
request_id = "duplex-sid-native-repeated-word-e0-stage0"
assert data_plane.segment_text_delta(request_id, "好的") == "好的"
assert data_plane.segment_text_delta(request_id, "好的好的") == "好的"
def test_duplex_data_plane_text_delta_appends_distinct_non_prefix_segments():
data_plane = _test_data_plane()
request_id = "duplex-sid-native-rewrite-e0-stage0"
deltas = [
data_plane.segment_text_delta(request_id, "It's Canberra."),
data_plane.segment_text_delta(request_id, " Next question."),
]
assert deltas == ["It's Canberra.", " Next question."]
assert "".join(deltas) == "It's Canberra. Next question."
def test_duplex_data_plane_close_session_removes_encoded_request_state():
data_plane = _test_data_plane()
target_request_id = duplex_resource_request_id(
DuplexFence("sid-to-close", incarnation=1, epoch=2),
"stage0",
)
other_request_id = duplex_resource_request_id(
DuplexFence("sid-to-keep", incarnation=1, epoch=2),
"stage0",
)
data_plane.begin_request(target_request_id)
data_plane.begin_request(other_request_id)
data_plane.close_session("sid-to-close")
assert data_plane.has_request(target_request_id) is False
assert data_plane.has_request(other_request_id) is True
def test_duplex_auto_response_segment_complete_keeps_data_plane_request_open():
import numpy as np
class _ChatService:
model_config = _ModelConfig()
def create_audio(self, audio_obj):
return SimpleNamespace(audio_data=f"wav-{int(audio_obj.audio_tensor.shape[0])}")
data_plane = _test_data_plane()
request_id = "duplex-duplex-sess-stage0"
session = DuplexSession(
session_id="sid-auto-respond",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.begin_response()
session.mark_audio_sent(1280)
data_plane.slice_cumulative_audio(request_id, np.zeros(30720, dtype=np.float32))
output = SimpleNamespace(
request_id=request_id,
finished=True,
outputs=[SimpleNamespace(text="hello", multimodal_output={})],
multimodal_output={
"audio": np.zeros(30720, dtype=np.float32),
"sr": 24000,
},
)
native_results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert len(native_results) == 1
assert native_results[0]["is_listen"] is True
assert native_results[0]["reason"] == "auto_response_segment_complete"
assert native_results[0]["data_plane_request_id"] == request_id
assert "abort_data_plane_request" not in native_results[0]
def test_duplex_auto_response_prior_playback_does_not_abort_before_current_response():
data_plane = _test_data_plane()
request_id = "duplex-duplex-sess-stage0"
session = DuplexSession(
session_id="sid-auto-respond-next-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.mark_audio_sent(1280)
session.acknowledge_playback(played_ms=1280, committed_ms=1280)
output = SimpleNamespace(
request_id=request_id,
finished=True,
outputs=[SimpleNamespace(text="", multimodal_output={})],
multimodal_output={},
)
native_results = _project_data_plane(data_plane, {"data_plane_outputs": [output]}, session=session)
assert native_results == []
def test_duplex_auto_response_text_only_flush_does_not_consume_audio_transcript():
import numpy as np
data_plane = _test_data_plane()
request_id = "duplex-duplex-sess-stage0"
session = DuplexSession(
session_id="sid-auto-respond-delayed-audio",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
text = "你好呀,有什莫问的吗?"
text_only_output = SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[SimpleNamespace(text=text, multimodal_output={})],
multimodal_output={"sr": 24000},
)
assert _project_data_plane(data_plane, {"data_plane_outputs": [text_only_output]}, session=session) == []
audio_output = SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[SimpleNamespace(text=text, multimodal_output={})],
multimodal_output={
"audio": np.zeros(24000, dtype=np.float32),
"sr": 24000,
},
)
native_results = _project_data_plane(data_plane, {"data_plane_outputs": [audio_output]}, session=session)
assert len(native_results) == 1
assert native_results[0]["audio_data"].startswith("wav-")
assert native_results[0]["text"] == text
@pytest.mark.asyncio
async def test_duplex_auto_response_empty_terminal_does_not_create_empty_response():
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
request_id = "duplex-sid-native-empty-terminal-e0-stage0"
session = DuplexSession(
session_id="sid-native-empty-terminal",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.mark_audio_sent(1280)
sent: list[dict[str, Any]] = []
async def send_json(payload: dict[str, Any]) -> None:
sent.append(payload)
output = SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[SimpleNamespace(text="hello", multimodal_output={})],
multimodal_output={
"sr": 24000,
"meta": {"turn_end": True, "duplex_turn_id": 0},
},
)
native_results = list(
handler._serving_runtime_adapter.data_plane.project(
{"data_plane_outputs": [output]},
context=_data_plane_context(session),
)
)
assert len(native_results) == 1
assert native_results[0]["end_of_turn"] is True
assert native_results[0]["model_turn_id"] == 0
for native_result in native_results:
await handler._send_one_native_duplex_event(send_json, native_result, session=session)
assert "response.created" not in [m.get("type") for m in sent]
assert "response.done" not in [m.get("type") for m in sent]
assert session.turn_id == 1
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_plane_state():
session_id = "sid-native-cleanup"
request_id = f"duplex-{session_id}-e0-stage0"
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
handler._serving_runtime_adapter.data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32))
handler._serving_runtime_adapter.data_plane.segment_text_delta(request_id, "hello")
handler._serving_runtime_adapter.data_plane.mark_terminal(request_id)
native = MiniCPMO45ServingSessionState()
handler._serving_runtime_adapter.session_states[session_id] = native
event = _native_session_create(session_id)
event["session"]["extra_body"]["auto_response"] = True
ws = TimedWebSocket()
ws.put(event)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert not handler._serving_runtime_adapter.data_plane.has_request(request_id)
assert session_id not in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
async def test_duplex_chat_audio_stream_uses_output_audio_delta_event():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
session = DuplexSession(session_id="sid-chat-audio", config=DuplexSessionConfig())
response_id = session.begin_response()
sent: list[dict[str, Any]] = []
async def send_json(data: dict[str, Any]) -> None:
sent.append(data)
await handler._emit_chat_payload(
session,
{
"modality": "audio",
"choices": [
{
"delta": {
"content": "AAAA",
}
}
],
},
session.epoch,
response_id,
send_json,
)
assert sent == [
{
"type": "response.output_audio.delta",
"session_id": "sid-chat-audio",
"response_id": response_id,
"epoch": 0,
"audio": "AAAA",
"format": "wav",
}
]
2026-08-29 19:56:37 +08:00
@pytest.mark.asyncio
async def test_duplex_chat_stage_metrics_use_latest_streaming_snapshot():
class SnapshotChatService(FakeChatService):
async def create_chat_completion(self, request, raw_request=None):
del raw_request
self.seen_request_ids.append(request.request_id)
snapshots = [
{"num_tokens_out": 1, "vllm_tpot_ms": 0.0, "vllm_itls_ms": []},
{"num_tokens_out": 2, "vllm_tpot_ms": 10.0, "vllm_itls_ms": [10.0]},
{"num_tokens_out": 3, "vllm_tpot_ms": 11.0, "vllm_itls_ms": [10.0, 12.0]},
]
async def generate():
for index, snapshot in enumerate(snapshots):
payload = {
"modality": "audio",
"choices": [{"delta": {"content": f"audio-{index}"}}],
"metrics": {"stage_metrics": {"0": snapshot}},
}
yield f"data: {json.dumps(payload)}\n\n"
return generate()
handler = OmniDuplexSessionHandler(
chat_service=SnapshotChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-chat-snapshots",
config=DuplexSessionConfig(instructions="speak"),
)
sent: list[dict[str, Any]] = []
async def send_json(data: dict[str, Any]) -> None:
sent.append(data)
await handler._run_response(session, send_json)
audio_deltas = [event for event in sent if event.get("type") == "response.output_audio.delta"]
done = next(event for event in sent if event.get("type") == "response.done")
assert all("vllm_omni" not in event for event in audio_deltas)
assert done["vllm_omni"]["stage_metrics"]["0"] == {
"num_tokens_out": 3,
"vllm_tpot_ms": 11.0,
"vllm_itls_ms": [10.0, 12.0],
}
@pytest.mark.asyncio
async def test_duplex_chat_rejects_unknown_output_modality():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(session_id="sid-chat-unknown", config=DuplexSessionConfig())
response_id = session.begin_response()
sent: list[dict[str, Any]] = []
async def send_json(data: dict[str, Any]) -> None:
sent.append(data)
await handler._emit_chat_payload(
session,
{
"modality": "video",
"choices": [{"delta": {"content": "not-text"}}],
},
session.epoch,
response_id,
send_json,
)
assert sent == [
{
"type": "error",
"session_id": session.session_id,
"response_id": response_id,
"epoch": session.epoch,
"code": "unsupported_response_modality",
"error": "Unsupported chat response modality: video",
}
]
assert session.assistant_text_buffer == ()
@pytest.mark.asyncio
async def test_minicpmo_model_name_does_not_auto_enable_experimental_native_duplex():
event = _session_create("sid-minicpmo-default-protocol")
event["session"]["model"] = "openbmb/MiniCPM-o-4_5"
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(event)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
created = next(m for m in ws.sent if m.get("type") == "session.created")
capabilities = created["session"]["capabilities"]
assert capabilities["implementation_level"] == "serving_session_adapter"
assert capabilities["supports_input_append"] is False
@pytest.mark.asyncio
async def test_duplex_handler_aborts_current_chat_request_id_on_barge_in():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") == "response.created":
ws.put({"type": "input.cancel", "reason": "test_barge_in"})
ws = TimedWebSocket(on_send=on_send)
ws.put(_session_create("sid-a"))
ws.put({"type": "input.text.append", "text": "hello"})
ws.put({"type": "input.commit"})
await handler.handle_session(ws)
assert ws.accepted
assert "response.created" in ws.sent_types()
assert "audio.cancelled" in ws.sent_types()
assert chat_service.seen_request_ids == ["duplex-sid-a-0-1"]
assert engine.aborted == ["chatcmpl-duplex-sid-a-0-1"]
assert engine.internal_abort_batches == [["chatcmpl-duplex-sid-a-0-1"]]
@pytest.mark.asyncio
async def test_duplex_handler_runtime_open_failure_is_reported_to_client():
engine = FakeEngineClient(fail_open=True)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-open-fail"))
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "runtime_open_failed"
assert engine.opened == []
@pytest.mark.asyncio
async def test_duplex_handler_preserves_typed_admission_error():
class AdmissionRejectingEngine(FakeEngineClient):
async def open_duplex_session_async(self, session_id: str, **kwargs):
del kwargs
raise DuplexControlRequestError(
{
"operation": "open",
"session_id": session_id,
"error": {
"code": "resource_exhausted",
"message": "duplex session capacity exhausted",
"retryable": True,
},
"accepted_fence": None,
"lease_generation": None,
}
)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(AdmissionRejectingEngine()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_session_create("sid-admission-rejected"))
await handler.handle_session(ws)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "resource_exhausted"
assert ws.sent[0]["retryable"] is True
@pytest.mark.asyncio
async def test_duplex_cancel_reports_playback_committed_cursor():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") == "response.created":
ws.put({"type": "playback.ack", "played_ms": 1200, "committed_ms": 1000})
ws.put({"type": "input.cancel", "reason": "test_barge_in"})
ws = TimedWebSocket(on_send=on_send)
ws.put(_session_create("sid-playback"))
ws.put({"type": "input.text.append", "text": "hello"})
ws.put({"type": "input.commit"})
await handler.handle_session(ws)
ack = next(m for m in ws.sent if m.get("type") == "playback.acknowledged")
cancelled = next(m for m in ws.sent if m.get("type") == "audio.cancelled")
assert ack["playback"]["played_ms"] == 1200
assert ack["playback"]["committed_ms"] == 1000
assert cancelled["committed_ms"] == 1000
assert cancelled["epoch"] == 1
@pytest.mark.asyncio
async def test_playback_ack_response_id_selects_matching_pending_history_item():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-playback-response-id",
config=DuplexSessionConfig(),
)
first_item_id = "item_resp-first"
second_item_id = "item_resp-second"
session.stage_pending_history_item(
first_item_id,
{"role": "assistant", "content": "first response"},
)
session.stage_pending_history_item(
second_item_id,
{"role": "assistant", "content": "second response"},
)
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": "resp-second",
"played_ms": 1000,
"committed_ms": 1000,
},
ws.send_json,
)
ack = next(message for message in ws.sent if message.get("type") == "playback.acknowledged")
assert ack["item_id"] == second_item_id
assert ack["history_committed"] is True
assert first_item_id in session.pending_history_item_ids
assert second_item_id not in session.pending_history_item_ids
@pytest.mark.asyncio
async def test_playback_ack_recovers_missing_response_history_registration():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-playback-missing-registration",
config=DuplexSessionConfig(playback_commit_policy="ack_only"),
)
response_id = session.begin_response()
session.append_assistant_text("response text")
session.mark_audio_sent(1000, text_chars=len("response text"))
session.end_response(commit_text=True, preserve_request=True)
assert not session.history_item_ids
assert f"item_{response_id}" in session.pending_history_item_ids
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": f"item_{response_id}",
"played_ms": 1000,
"committed_ms": 1000,
},
ws.send_json,
)
ack = next(message for message in ws.sent if message.get("type") == "playback.acknowledged")
assert ack["history_committed"] is True
assert f"item_{response_id}" in session.history_item_ids
assert session.history[-1] == {"role": "assistant", "content": "response text"}
@pytest.mark.asyncio
async def test_playback_ack_rejects_response_after_followup_user_commit():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-playback-after-followup-user",
config=DuplexSessionConfig(playback_commit_policy="ack_only"),
)
session.append_text("user A")
session.commit_user_input()
response_id = session.begin_response()
session.append_assistant_text("assistant A")
session.mark_audio_sent(1000, text_chars=len("assistant A"))
session.end_response(commit_text=False, preserve_request=True)
session.append_text("user B")
session.commit_user_input()
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": f"item_{response_id}",
"played_ms": 1000,
"committed_ms": 1000,
},
ws.send_json,
)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "playback_ack_too_late"
assert session.history == (
{"role": "user", "content": "user A"},
{"role": "user", "content": "user B"},
)
@pytest.mark.asyncio
async def test_precommit_zero_audio_ack_reserves_history_for_response_that_finishes_after_commit():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-zero-audio-checkpoint",
config=DuplexSessionConfig(playback_commit_policy="ack_only"),
)
session.mark_user_input_activity()
response_id = session.begin_response()
item_id = f"item_{response_id}"
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": item_id,
"played_ms": 0,
"committed_ms": 0,
},
ws.send_json,
)
session.commit_native_audio_input(transcript="later user input")
session.append_assistant_text("hello world")
session.mark_audio_sent(1000, text_chars=len("hello world"))
committed_message = session.end_response(commit_text=True, preserve_request=True)
session.register_history_item(item_id, committed_message)
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": item_id,
"played_ms": 1000,
"committed_ms": 1000,
},
ws.send_json,
)
assert not any(message.get("type") == "error" for message in ws.sent)
assert session.history == (
{"role": "assistant", "content": "hello world"},
{
"role": "user",
"content": [
{
"type": "audio_url",
"audio_url": {"url": "native-duplex:input-audio"},
"transcript": "later user input",
}
],
"transcript": "later user input",
},
)
@pytest.mark.asyncio
async def test_precommit_partial_ack_extends_history_after_response_finishes():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-partial-checkpoint",
config=DuplexSessionConfig(playback_commit_policy="ack_only"),
)
session.mark_user_input_activity()
response_id = session.begin_response()
item_id = f"item_{response_id}"
session.append_assistant_text("hello world")
session.mark_audio_sent(1000, text_chars=len("hello world"))
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": item_id,
"played_ms": 500,
"committed_ms": 500,
},
ws.send_json,
)
session.commit_native_audio_input(transcript="later user input")
committed_message = session.end_response(commit_text=True, preserve_request=True)
session.register_history_item(item_id, committed_message)
assert session.history[0] == {"role": "assistant", "content": "hello"}
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": item_id,
"played_ms": 1000,
"committed_ms": 1000,
},
ws.send_json,
)
assert not any(message.get("type") == "error" for message in ws.sent)
assert session.history[0] == {"role": "assistant", "content": "hello world"}
assert session.history[1]["role"] == "user"
@pytest.mark.asyncio
async def test_explicit_truncate_seals_snapshot_against_later_playback_ack():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-hard-playback-truncate",
config=DuplexSessionConfig(playback_commit_policy="ack_only"),
)
response_id = session.begin_response()
item_id = f"item_{response_id}"
session.append_assistant_text("hello world")
session.mark_audio_sent(1000, text_chars=len("hello world"))
session.end_response(commit_text=False, preserve_request=True)
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": item_id,
"played_ms": 500,
"committed_ms": 500,
"truncate": True,
},
ws.send_json,
)
assert session.history == ({"role": "assistant", "content": "hello"},)
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": response_id,
"item_id": item_id,
"played_ms": 1000,
"committed_ms": 1000,
},
ws.send_json,
)
assert not any(message.get("type") == "error" for message in ws.sent)
assert session.history == ({"role": "assistant", "content": "hello"},)
@pytest.mark.asyncio
async def test_cancel_does_not_append_old_assistant_after_followup_user():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-cancel-after-followup-user",
config=DuplexSessionConfig(playback_commit_policy="ack_only"),
)
session.append_text("user A")
session.commit_user_input()
session.begin_response()
session.append_assistant_text("assistant A")
session.mark_audio_sent(1000, text_chars=len("assistant A"))
session.acknowledge_playback(500, 500)
session.append_text("user B")
session.commit_user_input()
ws = TimedWebSocket()
cancelled = await handler._cancel_active_response(session, None, ws.send_json, reason="new_response")
assert cancelled is True
assert session.history == (
{"role": "user", "content": "user A"},
{"role": "user", "content": "user B"},
)
@pytest.mark.asyncio
async def test_playback_ack_rejects_item_from_another_response():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(session_id="sid-playback-mismatch", config=DuplexSessionConfig())
first_response_id = session.begin_response()
session.append_assistant_text("first response")
session.mark_audio_sent(1000)
session.end_response(commit_text=False, preserve_request=True)
session.register_history_item(f"item_{first_response_id}", None)
second_response_id = session.begin_response()
session.mark_audio_sent(1000)
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": second_response_id,
"item_id": f"item_{first_response_id}",
"played_ms": 500,
"committed_ms": 500,
},
ws.send_json,
)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "playback_item_mismatch"
assert session.active_response_id == second_response_id
assert session.playback.as_dict() == {
"generated_ms": 1000,
"sent_ms": 1000,
"played_ms": 0,
"committed_ms": 0,
}
@pytest.mark.asyncio
async def test_playback_ack_rejects_user_history_item():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(session_id="sid-playback-user-item", config=DuplexSessionConfig())
user_message = {"role": "user", "content": "user input"}
session.append_history_message(user_message)
session.register_history_item("item_user", user_message)
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": "user",
"item_id": "item_user",
"played_ms": 500,
"committed_ms": 500,
},
ws.send_json,
)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "playback_item_not_found"
assert session.history == ({"role": "user", "content": "user input"},)
@pytest.mark.asyncio
async def test_late_playback_ack_recovers_response_local_history_after_next_response_begins():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(session_id="sid-playback-response-local", config=DuplexSessionConfig())
first_response_id = session.begin_response()
session.append_assistant_text("first response")
session.mark_audio_sent(1000, text_chars=len("first response"))
session.end_response(commit_text=False, preserve_request=True)
second_response_id = session.begin_response()
session.mark_audio_sent(2000, text_chars=len("second response"))
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": first_response_id,
"item_id": f"item_{first_response_id}",
"played_ms": 1000,
"committed_ms": 1000,
},
ws.send_json,
)
ack = next(message for message in ws.sent if message.get("type") == "playback.acknowledged")
assert ack["history_committed"] is True
assert session.history[-1] == {"role": "assistant", "content": "first response"}
assert session.active_response_id == second_response_id
assert session.playback.as_dict() == {
"generated_ms": 2000,
"sent_ms": 2000,
"played_ms": 0,
"committed_ms": 0,
}
@pytest.mark.asyncio
async def test_late_playback_ack_does_not_advance_new_response_cursor():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-late-playback-ack",
config=DuplexSessionConfig(),
)
first_response_id = session.begin_response()
session.append_assistant_text("first response")
session.mark_audio_sent(11480, text_chars=len("first response"))
session.end_response(commit_text=False, preserve_request=True)
session.register_history_item(f"item_{first_response_id}", None)
second_response_id = session.begin_response()
session.mark_audio_sent(2200, text_chars=len("second response"))
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": first_response_id,
"item_id": f"item_{first_response_id}",
"played_ms": 11480,
"committed_ms": 11480,
},
ws.send_json,
)
assert session.active_response_id == second_response_id
assert session.playback.as_dict() == {
"generated_ms": 2200,
"sent_ms": 2200,
"played_ms": 0,
"committed_ms": 0,
}
ack = next(message for message in ws.sent if message.get("type") == "playback.acknowledged")
assert ack["playback"] == {
"generated_ms": 11480,
"sent_ms": 11480,
"played_ms": 11480,
"committed_ms": 11480,
}
assert first_response_id not in session.response_playbacks
assert session.response_playbacks[second_response_id] == session.playback
@pytest.mark.asyncio
async def test_late_playback_ack_truncates_history_with_matching_response_cursor():
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(FakeEngineClient()),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-late-playback-history",
config=DuplexSessionConfig(),
)
first_response_id = session.begin_response()
session.append_assistant_text("abcdefghij")
session.mark_audio_sent(1000)
session.end_response(commit_text=False, preserve_request=True)
session.register_history_item(f"item_{first_response_id}", None)
session.begin_response()
session.mark_audio_sent(10000)
ws = TimedWebSocket()
await handler._handle_playback_ack(
session,
{
"type": "playback.ack",
"response_id": first_response_id,
"item_id": f"item_{first_response_id}",
"played_ms": 500,
"committed_ms": 500,
},
ws.send_json,
)
assert session.history[-1] == {"role": "assistant", "content": "abcde"}
assert first_response_id in session.response_playbacks
@pytest.mark.asyncio
async def test_cancel_active_response_commits_only_played_assistant_history():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
session = DuplexSession(
session_id="sid-playback-partial",
config=DuplexSessionConfig(playback_commit_policy=DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value),
)
response_id = session.begin_response()
session.append_assistant_text("hello world")
session.mark_audio_sent(1000, text_chars=len("hello world"))
session.acknowledge_playback(played_ms=500, committed_ms=500)
cancelled = await handler._cancel_active_response(session, None, ws.send_json, reason="barge_in")
assert cancelled is True
assert session.epoch == 1
assert session.history == ({"role": "assistant", "content": "hello"},)
assert session.history_item_ids[f"item_{response_id}"] == session.history[0]
event = next(m for m in ws.sent if m.get("type") == "audio.cancelled")
assert event["committed_ms"] == 500
@pytest.mark.asyncio
async def test_cancel_active_response_keeps_truncated_history_in_serving():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
session = DuplexSession(
session_id="sid-playback-runtime-rebuild",
config=DuplexSessionConfig(playback_commit_policy=DuplexPlaybackCommitPolicy.COMMIT_ALL_ON_DONE.value),
)
response_id = session.begin_response()
session.append_assistant_text("hello world")
session.mark_audio_sent(1000, text_chars=len("hello world"))
session.acknowledge_playback(played_ms=500, committed_ms=500)
cancelled = await handler._cancel_active_response(
session,
None,
ws.send_json,
reason="barge_in",
)
assert cancelled is True
assert session.history == ({"role": "assistant", "content": "hello"},)
assert session.history_item_ids[f"item_{response_id}"] == session.history[0]
assert engine.signals == []
assert "error" not in ws.sent_types()
@pytest.mark.asyncio
async def test_cancel_active_native_data_plane_request_aborts_stage_request_id():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
session = DuplexSession(session_id="sid-native-abort", config=DuplexSessionConfig())
session.bind_request("duplex-sid-native-abort-e0-stage0-s1")
cancelled = await handler._cancel_active_response(session, None, ws.send_json, reason="barge_in")
assert cancelled is True
assert engine.aborted == ["duplex-sid-native-abort-e0-stage0-s1"]
assert engine.internal_abort_batches == [["duplex-sid-native-abort-e0-stage0-s1"]]
assert "audio.cancelled" in ws.sent_types()
@pytest.mark.asyncio
async def test_duplex_handler_explicit_close_closes_runtime_once_with_client_reason():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-close"))
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ws.sent_types() == ["session.created", "session.closed"]
assert engine.closed == [("sid-close", "session_close")]
@pytest.mark.asyncio
async def test_realtime_disconnect_detaches_resumable_session_without_runtime_close():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket(receive_timeout_s=0.01)
ws.put(_native_realtime_session_update("sid-resumable-disconnect"))
await handler.handle_realtime_session(ws)
created = next(message for message in ws.sent if message.get("type") == "session.created")
assert created["session"]["id"] == "sid-resumable-disconnect"
assert isinstance(created["resume_token"], str)
assert created["resume_token"]
assert created["attachment_generation"] == 1
assert engine.closed == []
assert engine.touched == [
("sid-resumable-disconnect", DuplexLeaseActivity.DETACH),
]
assert handler._registry.get("sid-resumable-disconnect") is not None
assert "sid-resumable-disconnect" in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
async def test_realtime_initial_resume_token_delivery_failure_closes_unrecoverable_session():
class FailingCreatedWebSocket(TimedWebSocket):
async def send_json(self, data: dict[str, Any]):
if data.get("type") == "session.created":
raise WebSocketDisconnect(code=1006)
await super().send_json(data)
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = FailingCreatedWebSocket(receive_timeout_s=0.01)
ws.put(_native_realtime_session_update("sid-created-delivery-failure"))
await handler.handle_realtime_session(ws)
assert engine.closed == [("sid-created-delivery-failure", "disconnect")]
assert handler._registry.get("sid-created-delivery-failure") is None
assert "sid-created-delivery-failure" not in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
async def test_realtime_send_after_created_disconnect_preserves_resumable_session():
class ClosedAfterCreatedWebSocket(TimedWebSocket):
def __init__(self) -> None:
super().__init__(receive_timeout_s=0.01)
self.transport_closed = False
async def send_json(self, data: dict[str, Any]):
if self.transport_closed:
raise RuntimeError(
"Unexpected ASGI message 'websocket.send', "
"after sending 'websocket.close' or response already completed."
)
await super().send_json(data)
if data.get("type") == "session.created":
self.transport_closed = True
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = ClosedAfterCreatedWebSocket()
ws.put(_native_realtime_session_update("sid-created-then-disconnected"))
await handler.handle_realtime_session(ws)
assert engine.closed == []
assert engine.touched == [
("sid-created-then-disconnected", DuplexLeaseActivity.DETACH),
]
assert handler._registry.get("sid-created-then-disconnected") is not None
assert "sid-created-then-disconnected" in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
async def test_realtime_resume_rotates_token_replays_and_preserves_runtime_identity():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
first = TimedWebSocket(receive_timeout_s=0.01)
first.put(_native_realtime_session_update("sid-realtime-resume"))
first.put({"type": "session.heartbeat"})
await handler.handle_realtime_session(first)
created = next(message for message in first.sent if message.get("type") == "session.created")
heartbeat = next(message for message in first.sent if message.get("type") == "session.heartbeat_ack")
token = created["resume_token"]
incarnation = created["incarnation"]
second = TimedWebSocket(receive_timeout_s=0.1)
second.query_params = {
"model": "openbmb/MiniCPM-o-4_5",
"native_duplex": "1",
"resume": "1",
}
second.put(
{
"type": "session.resume",
"session_id": "sid-realtime-resume",
"incarnation": incarnation,
"resume_token": token,
"last_received_server_event_seq": heartbeat["server_event_seq"] - 1,
}
)
second.put({"type": "session.close"})
await handler.handle_realtime_session(second)
resumed = next(message for message in second.sent if message.get("type") == "session.resumed")
replayed = [
message
for message in second.sent
if message.get("type") == "session.heartbeat_ack"
and message.get("server_event_seq") == heartbeat["server_event_seq"]
]
assert resumed["session_id"] == "sid-realtime-resume"
assert resumed["incarnation"] == incarnation
assert resumed["attachment_generation"] == 2
assert resumed["resume_token"] != token
assert replayed == [heartbeat]
assert engine.opened == ["sid-realtime-resume"]
assert engine.resumed == [("sid-realtime-resume", 0)]
assert engine.resume_fences[0] == engine.opened_fences[0]
assert engine.closed == [("sid-realtime-resume", "session_close")]
@pytest.mark.asyncio
async def test_realtime_resume_preserves_append_tail_order_across_connections():
class SlowFirstAppendEngine(FakeEngineClient):
def __init__(self) -> None:
super().__init__()
self.first_append_started = asyncio.Event()
self.second_append_started = asyncio.Event()
self.release_first_append = asyncio.Event()
self.append_start_order: list[int] = []
async def append_duplex_input_async(self, session_id: str, **kwargs):
call_index = len(self.append_start_order) + 1
self.append_start_order.append(call_index)
if call_index == 1:
self.first_append_started.set()
await self.release_first_append.wait()
else:
self.second_append_started.set()
kwargs.pop("expected_epoch", None)
return await super().append_duplex_input_async(session_id, **kwargs)
engine = SlowFirstAppendEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
first = TimedWebSocket(receive_timeout_s=1.0)
session_update = _native_realtime_session_update("sid-resume-append-order")
session_update["session"]["extra_body"]["auto_response"] = True
first.put(session_update)
first_task = asyncio.create_task(handler.handle_realtime_session(first))
for _ in range(100):
created = next((message for message in first.sent if message.get("type") == "session.created"), None)
if created is not None:
break
await asyncio.sleep(0.01)
else:
pytest.fail("resumable session did not open")
assert created is not None
session = handler._registry.get("sid-resume-append-order")
assert session is not None
native_state = handler._runtime_session_state(session)
first_scheduler = native_state.silence_continuation_scheduler
assert first_scheduler is not None
first.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(16000),
"sample_rate_hz": 16000,
"is_speech": True,
}
)
await asyncio.wait_for(engine.first_append_started.wait(), timeout=1)
await first.close(code=1006)
await asyncio.wait_for(first_task, timeout=1)
second = TimedWebSocket(receive_timeout_s=0.5)
second.query_params = {
"model": "openbmb/MiniCPM-o-4_5",
"native_duplex": "1",
"resume": "1",
}
second.put(
{
"type": "session.resume",
"session_id": "sid-resume-append-order",
"incarnation": created["incarnation"],
"resume_token": created["resume_token"],
"last_received_server_event_seq": created.get("server_event_seq", 0),
}
)
second_task = asyncio.create_task(handler.handle_realtime_session(second))
for _ in range(100):
if "session.resumed" in second.sent_types():
break
await asyncio.sleep(0.01)
else:
pytest.fail("session did not resume")
assert native_state.silence_continuation_scheduler is not first_scheduler
second.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(16000),
"sample_rate_hz": 16000,
"is_speech": True,
}
)
try:
await asyncio.sleep(0.05)
assert not engine.second_append_started.is_set()
finally:
engine.release_first_append.set()
await asyncio.wait_for(engine.second_append_started.wait(), timeout=1)
second.put({"type": "session.close"})
await asyncio.wait_for(second_task, timeout=2)
assert engine.append_start_order == [1, 2]
@pytest.mark.asyncio
async def test_realtime_takeover_ignores_late_detach_from_replaced_attachment():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
first = TimedWebSocket(receive_timeout_s=1.0)
first.put(_native_realtime_session_update("sid-takeover"))
first_task = asyncio.create_task(handler.handle_realtime_session(first))
for _ in range(100):
created = next((message for message in first.sent if message.get("type") == "session.created"), None)
if created is not None:
break
await asyncio.sleep(0.001)
else:
raise AssertionError("first attachment did not create the session")
second = TimedWebSocket(receive_timeout_s=0.03)
second.put(
{
"type": "session.resume",
"session_id": "sid-takeover",
"incarnation": created["incarnation"],
"resume_token": created["resume_token"],
"last_received_server_event_seq": 0,
}
)
await handler.handle_realtime_session(second)
await first_task
detach_touches = [item for item in engine.touched if item[1] is DuplexLeaseActivity.DETACH]
assert detach_touches == [("sid-takeover", DuplexLeaseActivity.DETACH)]
assert "session.replaced" in first.sent_types()
assert engine.resumed == [("sid-takeover", 0)]
assert engine.closed == []
@pytest.mark.asyncio
async def test_realtime_takeover_does_not_clear_new_attachment_pending_turns():
class DelayedReplacementCloseWebSocket(TimedWebSocket):
def __init__(self) -> None:
super().__init__(receive_timeout_s=1.0)
self.replacement_close_started = asyncio.Event()
self.release_replacement_close = asyncio.Event()
async def close(self, code: int = 1000, reason: str | None = None) -> None:
if not self.replacement_close_started.is_set():
self.replacement_close_started.set()
await self.release_replacement_close.wait()
await super().close(code=code, reason=reason)
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
first = DelayedReplacementCloseWebSocket()
first.put(_native_realtime_session_update("sid-takeover-pending-turn"))
first_task = asyncio.create_task(handler.handle_realtime_session(first))
for _ in range(100):
created = next((message for message in first.sent if message.get("type") == "session.created"), None)
if created is not None:
break
await asyncio.sleep(0.001)
else:
pytest.fail("first attachment did not create the session")
assert created is not None
second = TimedWebSocket(receive_timeout_s=1.0)
second.put(
{
"type": "session.resume",
"session_id": "sid-takeover-pending-turn",
"incarnation": created["incarnation"],
"resume_token": created["resume_token"],
"last_received_server_event_seq": 0,
}
)
second_task = asyncio.create_task(handler.handle_realtime_session(second))
await asyncio.wait_for(first.replacement_close_started.wait(), timeout=1)
session = handler._registry.get("sid-takeover-pending-turn")
assert session is not None
assert session.reserve_pending_turn(limit=2)
first.release_replacement_close.set()
await asyncio.wait_for(first_task, timeout=1)
assert session.pending_input_turns == 1
session.release_pending_turn()
second.put({"type": "session.close"})
await asyncio.wait_for(second_task, timeout=2)
@pytest.mark.asyncio
async def test_realtime_takeover_drops_input_queued_by_replaced_attachment():
class DelayedReplacementCloseWebSocket(TimedWebSocket):
def __init__(self) -> None:
super().__init__(receive_timeout_s=1.0)
self.replacement_close_started = asyncio.Event()
self.release_replacement_close = asyncio.Event()
self.stale_heartbeat_read = asyncio.Event()
async def receive_text(self) -> str:
payload = await super().receive_text()
if json.loads(payload).get("type") == "session.heartbeat":
self.stale_heartbeat_read.set()
return payload
async def close(self, code: int = 1000, reason: str | None = None) -> None:
if not self.replacement_close_started.is_set():
self.replacement_close_started.set()
await self.release_replacement_close.wait()
await super().close(code=code, reason=reason)
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
first = DelayedReplacementCloseWebSocket()
first.put(_native_realtime_session_update("sid-takeover-stale-input"))
first_task = asyncio.create_task(handler.handle_realtime_session(first))
for _ in range(100):
created = next((message for message in first.sent if message.get("type") == "session.created"), None)
if created is not None:
break
await asyncio.sleep(0.001)
else:
pytest.fail("first attachment did not create the session")
assert created is not None
second = TimedWebSocket(receive_timeout_s=1.0)
second.put(
{
"type": "session.resume",
"session_id": "sid-takeover-stale-input",
"incarnation": created["incarnation"],
"resume_token": created["resume_token"],
"last_received_server_event_seq": 0,
}
)
second_task = asyncio.create_task(handler.handle_realtime_session(second))
await asyncio.wait_for(first.replacement_close_started.wait(), timeout=1)
first.put({"type": "session.heartbeat"})
await asyncio.wait_for(first.stale_heartbeat_read.wait(), timeout=1)
first.release_replacement_close.set()
await asyncio.wait_for(first_task, timeout=1)
second.put({"type": "session.close"})
await asyncio.wait_for(second_task, timeout=2)
assert all(activity is not DuplexLeaseActivity.HEARTBEAT for _, activity in engine.touched)
assert "session.heartbeat_ack" not in second.sent_types()
@pytest.mark.asyncio
async def test_realtime_resume_rejects_invalid_token_before_engine_control():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
first = TimedWebSocket(receive_timeout_s=0.01)
first.put(_native_realtime_session_update("sid-invalid-resume"))
await handler.handle_realtime_session(first)
created = next(message for message in first.sent if message.get("type") == "session.created")
second = TimedWebSocket(receive_timeout_s=0.01)
second.put(
{
"type": "session.resume",
"session_id": "sid-invalid-resume",
"incarnation": created["incarnation"],
"resume_token": "invalid-token",
"last_received_server_event_seq": 0,
}
)
await handler.handle_realtime_session(second)
error = next(message for message in second.sent if message.get("type") == "error")
assert error["error"]["code"] == "invalid_resume_token"
assert engine.resumed == []
assert engine.closed == []
@pytest.mark.asyncio
async def test_realtime_resume_recovers_when_rotated_token_delivery_is_lost():
class FailingResumeWebSocket(TimedWebSocket):
async def send_json(self, data: dict[str, Any]):
if data.get("type") == "session.resumed":
raise RuntimeError("resume transport disappeared")
await super().send_json(data)
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
first = TimedWebSocket(receive_timeout_s=0.01)
first.put(_native_realtime_session_update("sid-token-delivery"))
await handler.handle_realtime_session(first)
created = next(message for message in first.sent if message.get("type") == "session.created")
resume_event = {
"type": "session.resume",
"session_id": "sid-token-delivery",
"incarnation": created["incarnation"],
"resume_token": created["resume_token"],
"last_received_server_event_seq": 0,
}
failed_transport = FailingResumeWebSocket(receive_timeout_s=0.01)
failed_transport.put(resume_event)
await handler.handle_realtime_session(failed_transport)
recovered_transport = TimedWebSocket(receive_timeout_s=0.1)
recovered_transport.put(resume_event)
recovered_transport.put({"type": "session.close"})
await handler.handle_realtime_session(recovered_transport)
assert "session.resumed" in recovered_transport.sent_types()
assert engine.resumed == [
("sid-token-delivery", 0),
("sid-token-delivery", 1),
]
assert engine.closed == [("sid-token-delivery", "session_close")]
@pytest.mark.asyncio
async def test_realtime_disconnect_grace_cancels_only_orphan_response():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
duplex_session_config=DuplexSessionRuntimeConfig(disconnect_grace_s=0.02),
)
ws = TimedWebSocket(receive_timeout_s=0.01)
ws.put(_native_realtime_session_update("sid-grace-response"))
await handler.handle_realtime_session(ws)
session = handler._registry.get("sid-grace-response")
assert session is not None
session.begin_response()
session.bind_request("orphan-request")
orphan_task = asyncio.create_task(asyncio.sleep(10))
handler._session_tasks[session.session_id].active_response_task = orphan_task
assert engine.closed == []
await asyncio.sleep(0.05)
assert engine.closed == []
assert engine.aborted == ["orphan-request"]
assert orphan_task.cancelled()
assert session.active_response_id is None
assert "sid-grace-response" in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
async def test_realtime_heartbeat_control_failure_is_non_terminal():
engine = FakeEngineClient(fail_touch=True)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket(receive_timeout_s=0.1)
ws.put(_native_realtime_session_update("sid-heartbeat-failure"))
ws.put({"type": "session.heartbeat"})
ws.put({"type": "session.close"})
await handler.handle_realtime_session(ws)
error = next(message for message in ws.sent if message.get("type") == "error")
assert error["error"]["code"] == "runtime_touch_failed"
assert engine.closed == [("sid-heartbeat-failure", "session_close")]
@pytest.mark.asyncio
async def test_realtime_idle_ttl_lifecycle_removes_detached_serving_projection():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket(receive_timeout_s=0.01)
ws.put(_native_realtime_session_update("sid-idle-expired"))
await handler.handle_realtime_session(ws)
fence = engine.opened_fences[0]
assert fence is not None
await engine.duplex_lifecycle_events.put(
DuplexSessionLifecycleMessage(
fence=fence,
session_id="sid-idle-expired",
event="expired",
reason="idle_ttl_expired",
lease_generation=1,
submitted_request_ids=[],
reserved_request_ids=[],
)
)
await asyncio.sleep(0.05)
assert handler._registry.get("sid-idle-expired") is None
assert "sid-idle-expired" not in handler._serving_runtime_adapter.session_states
assert "sid-idle-expired" not in handler._session_tasks
assert engine.closed == []
@pytest.mark.asyncio
async def test_realtime_resume_reports_resync_required_after_journal_ttl_gap():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
duplex_session_config=DuplexSessionRuntimeConfig(resume_replay_ttl_s=0.01),
)
first = TimedWebSocket(receive_timeout_s=0.01)
first.put(_native_realtime_session_update("sid-journal-gap"))
first.put({"type": "session.heartbeat"})
await handler.handle_realtime_session(first)
created = next(message for message in first.sent if message.get("type") == "session.created")
await asyncio.sleep(0.02)
second = TimedWebSocket(receive_timeout_s=0.01)
second.put(
{
"type": "session.resume",
"session_id": "sid-journal-gap",
"incarnation": created["incarnation"],
"resume_token": created["resume_token"],
"last_received_server_event_seq": 0,
}
)
await handler.handle_realtime_session(second)
assert "session.resync_required" in second.sent_types()
assert engine.resumed == []
assert engine.closed == []
@pytest.mark.asyncio
async def test_realtime_journal_overflow_degrades_to_live_resync_without_closing_runtime():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
duplex_session_config=DuplexSessionRuntimeConfig(resume_replay_max_bytes_per_session=128),
)
ws = TimedWebSocket(receive_timeout_s=0.01)
ws.put(_native_realtime_session_update("sid-journal-overflow"))
await handler.handle_realtime_session(ws)
assert "session.created" in ws.sent_types()
assert "session.resync_required" in ws.sent_types()
assert engine.closed == []
assert handler._registry.get("sid-journal-overflow") is not None
@pytest.mark.asyncio
async def test_realtime_input_backpressure_is_per_session_and_rejects_before_mutation():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
duplex_session_config=DuplexSessionRuntimeConfig(
max_pending_input_bytes_per_session=100,
),
)
first = TimedWebSocket(receive_timeout_s=0.1)
first.put(_native_realtime_session_update("sid-backpressure-a"))
for _ in range(2):
first.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(20),
"format": "pcm16",
"sample_rate_hz": 16000,
}
)
first.put({"type": "session.close"})
await handler.handle_realtime_session(first)
backpressure = [
event
for event in first.sent
if event.get("type") == "error"
and isinstance(event.get("error"), dict)
and event["error"].get("code") == "input_backpressure"
]
assert len(backpressure) == 1
second = TimedWebSocket(receive_timeout_s=0.1)
second.put(_native_realtime_session_update("sid-backpressure-b"))
second.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(20),
"format": "pcm16",
"sample_rate_hz": 16000,
}
)
second.put({"type": "session.close"})
await handler.handle_realtime_session(second)
assert not any(
event.get("type") == "error"
and isinstance(event.get("error"), dict)
and event["error"].get("code") == "input_backpressure"
for event in second.sent
)
@pytest.mark.asyncio
async def test_duplex_handler_idle_timeout_close_does_not_emit_runtime_control():
control_result = {
"operation": "close",
"session_id": "sid-disconnect",
"ok": True,
"unsupported_count": 1,
"error_count": 0,
"stage_results": [{"stage_id": 0, "replica_id": 0, "result": {"supported": False}}],
}
engine = FakeEngineClient(append_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=0.1)
ws = TimedWebSocket()
ws.put(_session_create("sid-disconnect"))
await handler.handle_session(ws)
assert engine.closed == [("sid-disconnect", "timeout")]
@pytest.mark.asyncio
async def test_idle_output_audio_clear_does_not_advance_runtime_fence():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_session_create("sid-idle-output-clear"))
ws.put({"type": "output_audio_buffer.clear"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
cancelled = next(event for event in ws.sent if event.get("type") == "audio.cancelled")
assert cancelled["cancelled_epoch"] == cancelled["epoch"] == 0
assert engine.signals == []
@pytest.mark.asyncio
async def test_duplex_handler_runtime_close_failure_is_reported_without_closed_ack():
engine = FakeEngineClient(fail_close=True)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-close-fail"))
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ws.sent_types() == ["session.created", "error"]
assert ws.sent[-1]["code"] == "runtime_close_failed"
@pytest.mark.asyncio
async def test_duplex_handler_control_close_failure_is_reported_without_closed_ack():
control_result = {
"operation": "close",
"session_id": "sid-control-close-fail",
"ok": False,
"unsupported_count": 0,
"error_count": 1,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {"supported": False, "error": "stage close failed"},
}
],
}
engine = FakeEngineClient(close_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-control-close-fail"))
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ws.sent_types() == ["session.created", "error"]
assert ws.sent[-1]["code"] == "runtime_close_failed"
assert ws.sent[-1]["runtime_control"]["error_count"] == 1
@pytest.mark.asyncio
async def test_duplex_cancel_without_active_response_clears_pending_input_and_acks():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-pending-cancel"))
ws.put({"type": "input.text.append", "text": "hello"})
ws.put({"type": "input.cancel", "reason": "user_cancel"})
ws.put({"type": "input.commit"})
await handler.handle_session(ws)
cancelled = next(m for m in ws.sent if m.get("type") == "input.cancelled")
assert cancelled["cancelled"] == {"text_chunks": 1, "audio_chunks": 0}
assert cancelled["epoch"] == 1
assert chat_service.seen_request_ids == []
assert "response.created" not in ws.sent_types()
@pytest.mark.asyncio
async def test_turn_signal_input_cancel_uses_epoch_fence_transition():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_session_create("sid-turn-signal-cancel"))
ws.put({"type": "input.text.append", "text": "discard me"})
ws.put({"type": "turn.signal", "event": "input.cancel"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert engine.signal_fences[-1] == DuplexFence("sid-turn-signal-cancel")
assert engine.signal_next_fences[-1] == DuplexFence("sid-turn-signal-cancel", epoch=1)
assert any(message.get("type") == "input.cancelled" for message in ws.sent)
@pytest.mark.asyncio
async def test_duplex_handler_local_turn_signal_does_not_round_trip_runtime():
engine = FakeEngineClient(fail_signal=True)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-signal-fail"))
ws.put({"type": "turn.signal", "event": "user_started"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
turn_event = next(m for m in ws.sent if m.get("type") == "turn.event")
assert turn_event["event"] == "user_started"
assert not engine.signals
assert "runtime_signal_failed" not in {m.get("code") for m in ws.sent}
@pytest.mark.asyncio
async def test_duplex_barge_in_aborts_active_response_when_runtime_signal_fails():
engine = FakeEngineClient(fail_signal_events={"barge_in"})
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") == "response.created":
ws.put({"type": "input.cancel", "reason": "test_barge_in"})
ws = TimedWebSocket(on_send=on_send)
ws.put(_session_create("sid-barge-signal-fail"))
ws.put({"type": "input.text.append", "text": "hello"})
ws.put({"type": "input.commit"})
await handler.handle_session(ws)
assert "audio.cancelled" in ws.sent_types()
assert engine.aborted == ["chatcmpl-duplex-sid-barge-signal-fail-0-1"]
assert engine.internal_abort_batches == [["chatcmpl-duplex-sid-barge-signal-fail-0-1"]]
error = next(m for m in ws.sent if m.get("type") == "error")
assert error["code"] == "runtime_signal_failed"
@pytest.mark.asyncio
async def test_duplex_handler_surfaces_stage_unsupported_result_to_client():
control_result = {
"operation": "open",
"session_id": "sid-unsupported",
"ok": True,
"unsupported_count": 1,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {"supported": False, "reason": "not implemented"},
}
],
}
engine = FakeEngineClient(open_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-unsupported"))
ws.put({"type": "session.close"})
await handler.handle_session(ws)
created = next(m for m in ws.sent if m.get("type") == "session.created")
assert created["runtime_control"]["unsupported_count"] == 1
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_append_without_format_defaults_to_pcm16():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-default-pcm16"))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(16000),
"sample_rate_hz": 16000,
"is_speech": True,
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert len(engine.appended) == 1
_, mode, payload, final = engine.appended[0]
assert mode == "append_audio_chunk"
assert final is False
assert isinstance(payload, dict)
assert payload["format"] == "pcm_f32le"
assert payload["sample_rate_hz"] == 16000
samples = np.frombuffer(base64.b64decode(payload["audio"]), dtype="<f4")
assert samples.shape == (16000,)
assert samples[:4].tolist() == pytest.approx([1000 / 32768.0] * 4)
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_rejects_invalid_sample_rate_without_append():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-invalid-rate"))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(1),
"sample_rate_hz": 16_000.5,
"is_speech": True,
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert engine.appended == []
error = next(event for event in ws.sent if event.get("type") == "error")
assert error["code"] == "bad_event"
@pytest.mark.asyncio
async def test_native_input_commit_does_not_send_noop_runtime_signal():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-commit-without-signal"))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(16000),
"sample_rate_hz": 16000,
"is_speech": True,
}
)
ws.put({"type": "input_audio_buffer.commit"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ("sid-commit-without-signal", "input.commit") not in engine.signals
@pytest.mark.asyncio
async def test_audio_clear_preserves_later_wire_order_append():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-clear-then-append"))
ws.put({"type": "input_audio_buffer.clear"})
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(16000),
"sample_rate_hz": 16000,
"is_speech": True,
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert len(engine.appended) == 1
@pytest.mark.asyncio
async def test_native_append_is_cancelled_by_later_wire_order_input_cancel(mocker):
class SlowAppendEngine(FakeEngineClient):
def __init__(self) -> None:
super().__init__()
self.append_started = asyncio.Event()
self.append_cancelled = asyncio.Event()
self.release_append = asyncio.Event()
async def append_duplex_input_async(
self,
session_id: str,
*,
mode: str,
payload: object,
operation_id: str | None = None,
final: bool = False,
timeout: float | None = None,
collect_outputs: bool = True,
fence: DuplexFence | None = None,
) -> Any:
self.append_started.set()
try:
await self.release_append.wait()
except asyncio.CancelledError:
self.append_cancelled.set()
raise
return await super().append_duplex_input_async(
session_id,
mode=mode,
payload=payload,
operation_id=operation_id,
final=final,
timeout=timeout,
collect_outputs=collect_outputs,
fence=fence,
)
rollback = mocker.spy(MiniCPMO45PcmAppendBuffer, "_rollback_reservation")
engine = SlowAppendEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-cancel-slow-append"))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm16_b64(16000),
"sample_rate_hz": 16000,
"is_speech": True,
}
)
ws.put({"type": "input.cancel"})
ws.put({"type": "session.close"})
handler_task = asyncio.create_task(handler.handle_session(ws))
await asyncio.wait_for(engine.append_started.wait(), timeout=1)
try:
await asyncio.wait_for(engine.append_cancelled.wait(), timeout=0.2)
except TimeoutError:
pass
finally:
engine.release_append.set()
await asyncio.wait_for(handler_task, timeout=1)
assert engine.append_cancelled.is_set()
rollback.assert_called_once()
assert engine.signal_fences[-1] == DuplexFence("sid-cancel-slow-append")
assert engine.signal_next_fences[-1] == DuplexFence("sid-cancel-slow-append", epoch=1)
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_append_control_error_does_not_emit_model_delta():
control_result = {
"operation": "append",
"session_id": "sid-native-append-error",
"ok": False,
"unsupported_count": 0,
"error_count": 1,
"stage_results": [
{
"stage_id": -1,
"replica_id": -1,
"result": {
"supported": False,
"error": "duplex_stage_handoff_target_missing:tts",
},
},
],
}
engine = FakeEngineClient(append_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-append-error"))
ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert "response.output_audio.delta" not in ws.sent_types()
error = next(m for m in ws.sent if m.get("type") == "error")
assert error["code"] == "runtime_append_failed"
assert error["runtime_control"]["error_count"] == 1
@pytest.mark.asyncio
async def test_duplex_handler_session_update_control_failure_rolls_back_config():
control_result = {
"operation": "signal",
"session_id": "sid-control-signal-fail",
"ok": False,
"unsupported_count": 0,
"error_count": 1,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {"supported": False, "error": "stage signal failed"},
}
],
}
engine = FakeEngineClient(signal_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-control-signal-fail"))
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"instructions": "must roll back"},
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert "session.updated" not in ws.sent_types()
error = next(m for m in ws.sent if m.get("type") == "error")
assert error["code"] == "runtime_signal_failed"
assert error["runtime_control"]["error_count"] == 1
signaled_config = engine.signal_session_configs[-1]
assert signaled_config is not None
assert signaled_config["instructions"] == "must roll back"
@pytest.mark.asyncio
async def test_native_session_update_commits_config_only_after_runtime_ack():
class BlockingUpdateEngine(FakeEngineClient):
def __init__(self):
super().__init__()
self.update_started = asyncio.Event()
self.release_update = asyncio.Event()
async def signal_duplex_turn_async(self, session_id: str, **kwargs):
if kwargs.get("event") == "session.update":
self.update_started.set()
await self.release_update.wait()
return await super().signal_duplex_turn_async(session_id, **kwargs)
engine = BlockingUpdateEngine()
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
ws.put(_native_realtime_session_update("sid-update-two-phase"))
ws.put(_server_vad_update("update-two-phase"))
handler_task = asyncio.create_task(handler.handle_session(ws, realtime_protocol=protocol))
await asyncio.wait_for(engine.update_started.wait(), timeout=1)
live_session = handler._registry.get("sid-update-two-phase")
assert live_session is not None
assert live_session.config.overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
assert protocol._pending_turn_detection_update is not None
assert protocol._server_vad is None
protocol.encode_outbound_event({"type": "error", "code": "unrelated", "error": "unrelated"})
assert protocol._pending_turn_detection_update is not None
engine.release_update.set()
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
updated = next(message for message in reversed(ws.sent) if message.get("type") == "session.updated")
assert updated["session"]["overlap_policy"] == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value
assert isinstance(protocol._server_vad, realtime_vad.SileroStreamingVAD)
@pytest.mark.parametrize(
("append_fails", "expected_code"),
[(True, "session_update_aborted"), (False, "instructions_update_unsupported")],
)
@pytest.mark.asyncio
async def test_realtime_vad_update_rejection_does_not_stall(append_fails, expected_code):
append_result = {"operation": "append", "ok": False, "error_count": 1} if append_fails else None
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient(append_result=append_result)))
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
create = _native_realtime_session_update("sid-rejected-vad-update")
create["session"]["extra_body"]["auto_response"] = append_fails
ws.put(create)
if append_fails:
ws.put({"type": "input_audio_buffer.append", "audio": _pcm_f32_b64(16_000), "format": "pcm_f32le"})
update = _server_vad_update("rejected-vad-update")
if not append_fails:
update["session"]["instructions"] = "You are now a pirate."
ws.put(update)
ws.put({"type": "session.close"})
await asyncio.wait_for(handler.handle_session(ws, realtime_protocol=protocol), timeout=2)
errors = [message["error"] for message in ws.sent if isinstance(message.get("error"), dict)]
error = next(error for error in errors if error.get("code") == expected_code)
assert error["event_id"] == "rejected-vad-update"
assert protocol._pending_turn_detection_update is None
assert protocol._server_vad is None
@pytest.mark.asyncio
async def test_native_session_update_waits_for_prior_append_effect():
class OrderedUpdateEngine(FakeEngineClient):
def __init__(self):
super().__init__()
self.append_started = asyncio.Event()
self.release_append = asyncio.Event()
self.update_started = asyncio.Event()
self.effects: list[str] = []
async def append_duplex_input_async(self, session_id: str, **kwargs):
self.effects.append("append.started")
self.append_started.set()
await self.release_append.wait()
kwargs.pop("expected_epoch", None)
result = await super().append_duplex_input_async(session_id, **kwargs)
self.effects.append("append.done")
return result
async def signal_duplex_turn_async(self, session_id: str, **kwargs):
if kwargs.get("event") == "session.update":
self.effects.append("session.update")
self.update_started.set()
return await super().signal_duplex_turn_async(session_id, **kwargs)
engine = OrderedUpdateEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-ordered-update"))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000, value=0.05),
"format": "pcm_f32le",
}
)
ws.put(
{
"type": "turn.signal",
"event": "session.update",
"payload": {"temperature": 0.2},
}
)
handler_task = asyncio.create_task(handler.handle_session(ws))
await asyncio.wait_for(engine.append_started.wait(), timeout=1)
try:
await asyncio.sleep(0.05)
assert not engine.update_started.is_set()
finally:
engine.release_append.set()
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert engine.effects.index("append.done") < engine.effects.index("session.update")
@pytest.mark.asyncio
async def test_failed_native_append_prevents_queued_append_from_running():
failed_result = {
"operation": "append",
"session_id": "sid-append-chain-failure",
"ok": False,
"unsupported_count": 0,
"error_count": 1,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {"supported": False, "error": "append failed"},
}
],
}
class BlockingFailureEngine(FakeEngineClient):
def __init__(self):
super().__init__(append_result=failed_result)
self.first_started = asyncio.Event()
self.release_first = asyncio.Event()
async def append_duplex_input_async(self, session_id: str, **kwargs):
if not self.appended:
self.first_started.set()
await self.release_first.wait()
kwargs.pop("expected_epoch", None)
return await super().append_duplex_input_async(session_id, **kwargs)
engine = BlockingFailureEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
create = _native_session_create("sid-append-chain-failure")
create["session"]["extra_body"]["auto_response"] = True
ws.put(create)
for _ in range(2):
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16_000, value=0.05),
"format": "pcm_f32le",
}
)
handler_task = asyncio.create_task(handler.handle_session(ws))
await asyncio.wait_for(engine.first_started.wait(), timeout=1)
engine.release_first.set()
await asyncio.sleep(0.1)
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert len(engine.appended) == 1
@pytest.mark.asyncio
async def test_failed_final_append_discards_retained_audio_without_retry():
failed_result = {
"operation": "append",
"session_id": "sid-final-append-retry",
"ok": False,
"unsupported_count": 0,
"error_count": 1,
"stage_results": [],
}
engine = FakeEngineClient(append_result=failed_result)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
def retry_after_runtime_error(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") == "error" and data.get("code") == "runtime_append_failed":
ws.put({"type": "response.create"})
ws = TimedWebSocket(on_send=retry_after_runtime_error)
ws.put(_native_session_create("sid-final-append-retry"))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(8_000, value=0.05),
"format": "pcm_f32le",
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
ws.put({"type": "response.create"})
handler_task = asyncio.create_task(handler.handle_session(ws))
for _ in range(100):
state = handler._serving_runtime_adapter.session_states.get("sid-final-append-retry")
if len(engine.appended) >= 1 and state is not None and state.committed_audio_payload is None:
break
await asyncio.sleep(0.01)
state = handler._serving_runtime_adapter.session_states["sid-final-append-retry"]
assert state.committed_audio_payload is None
assert state.committed_audio_reserved_bytes == 0
assert handler._registry.get("sid-final-append-retry").pending_input_bytes == 0
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert len(engine.appended) == 1
@pytest.mark.asyncio
async def test_duplex_handler_signal_unsupported_workers_with_data_plane_ack_is_not_fatal():
control_result = {
"operation": "signal",
"session_id": "sid-control-signal-data-plane",
"ok": False,
"unsupported_count": 2,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {"supported": False, "reason": "worker_duplex_signal_not_implemented"},
},
{
"stage_id": 1,
"replica_id": 0,
"result": {"supported": False, "reason": "worker_duplex_signal_not_implemented"},
},
{
"stage_id": -1,
"replica_id": -1,
"result": {
"supported": True,
"data_plane_signal": True,
"event": "barge_in",
"epoch": 1,
},
},
],
}
engine = FakeEngineClient(signal_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_session_create("sid-control-signal-data-plane"))
ws.put({"type": "turn.signal", "event": "barge_in"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert "runtime_signal_failed" not in {m.get("code") for m in ws.sent}
assert "input.cancelled" in ws.sent_types()
runtime_control = next(m for m in ws.sent if m.get("type") == "runtime.control")
assert runtime_control["result"]["unsupported_count"] == 2
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_open_unsupported_fails_session_create():
control_result = {
"operation": "open",
"session_id": "sid-native-busy",
"ok": True,
"unsupported_count": 1,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {"supported": False, "reason": "native_duplex_session_busy"},
}
],
}
engine = FakeEngineClient(control_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-busy"))
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "runtime_open_unsupported"
assert ws.sent[0]["runtime_control"]["unsupported_count"] == 1
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_rejects_engine_without_fence_contract():
class LegacyEngineClient(FakeEngineClient):
async def open_duplex_session_async( # type: ignore[override]
self,
session_id: str,
*,
session_mode: str = "duplex",
capabilities: dict[str, object] | None = None,
session_config: dict[str, object] | None = None,
timeout: float | None = None,
) -> Any:
return await super().open_duplex_session_async(
session_id,
session_mode=session_mode,
capabilities=capabilities,
session_config=session_config,
timeout=timeout,
)
engine = LegacyEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-no-fence-contract"))
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "runtime_contract_invalid"
assert "open_duplex_session_async" in ws.sent[0]["error"]
assert "fence" in ws.sent[0]["error"]
assert engine.opened == []
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_audio_append_does_not_retain_pending_audio():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-no-pending"))
short_chunk = base64.b64encode(b"\x00" * (800 * 4)).decode("ascii")
ws.put({"type": "input_audio_buffer.append", "audio": short_chunk, "format": "pcm_f32le"})
ws.put({"type": "input.cancel", "reason": "barge_in"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
cancelled = next(m for m in ws.sent if m.get("type") == "input.cancelled")
assert cancelled["cancelled"] == {"text_chunks": 0, "audio_chunks": 0}
assert cancelled["epoch"] == 1
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_rejects_text_append_before_runtime_call():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-text"))
ws.put({"type": "input.text.append", "text": "hello"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
error = next(m for m in ws.sent if m.get("type") == "error")
assert error["code"] == "native_text_append_unsupported"
assert engine.appended == []
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_separates_public_and_runtime_config(monkeypatch):
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
async def fake_resolve_ref_audio(_, *, model_config):
del model_config
return [0.25] * 1600, 16000
monkeypatch.setattr(
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio",
fake_resolve_ref_audio,
)
event = _native_session_create("sid-native-ref-audio", modalities=["text", "audio"])
event["session"]["ref_audio"] = "data:audio/wav;base64,AAAA"
ws = TimedWebSocket()
ws.put(event)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert ws.sent_types()[0] == "session.created"
opened_config = engine.opened_configs[0]
opened_runtime_config = engine.opened_runtime_configs[0]
extra_body = opened_config["extra_body"]
assert "ref_audio_data" not in extra_body
assert "duplex_stage_max_tokens" not in extra_body
assert opened_runtime_config["ref_audio_format"] == "pcm_f32le"
assert opened_runtime_config["ref_audio_sample_rate_hz"] == 16000
assert opened_runtime_config["duplex_stage_max_tokens"] == {"0": 20, "1": 8192}
assert opened_runtime_config["duplex_stage_sampling_params"]["0"]["top_k"] == 20
assert opened_runtime_config["duplex_stage_sampling_params"]["0"]["top_p"] == 0.8
# The Talker YAML carries upstream's min_new_token=50 for chat TTS; a duplex
# chunk is 26 codec samples, so check_stop would never release the request.
assert opened_runtime_config["duplex_stage_sampling_params"]["1"] == {"min_tokens": 0}
assert base64.b64decode(opened_runtime_config["ref_audio_data"]) == struct.pack("<1600f", *([0.25] * 1600))
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_rejects_client_runtime_config():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
event = _native_session_create("sid-native-private-config")
event["session"]["extra_body"]["duplex_stage_sampling_params"] = {"0": {"temperature": 0.0}}
ws = TimedWebSocket()
ws.put(event)
await handler.handle_session(ws)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "invalid_duplex_runtime_config"
assert engine.opened == []
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_audio_output_requires_ref_audio(monkeypatch):
monkeypatch.setattr(
MiniCPMO45NativeDuplexServingAdapter,
"_load_native_tokenizer",
staticmethod(lambda model_config: None),
)
event = _native_session_create(
"sid-native-ref-required",
modalities=["text", "audio"],
)
config = DuplexSessionConfig.from_event(event)
with pytest.raises(ServingRuntimeConfigError) as exc_info:
await MiniCPMO45NativeDuplexServingAdapter.prepare_runtime_config(
config,
model_config=SimpleNamespace(model="openbmb/MiniCPM-o-4_5"),
)
assert exc_info.value.code == "ref_audio_required"
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_open_session_maps_missing_ref_audio_to_typed_error():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(
_native_session_create(
"sid-native-ref-required-open",
modalities=["text", "audio"],
)
)
await handler.handle_session(ws)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "ref_audio_required"
assert engine.opened == []
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_text_only_omits_ref_audio_when_client_does_not_provide_it(monkeypatch):
monkeypatch.setattr(
MiniCPMO45NativeDuplexServingAdapter,
"_load_native_tokenizer",
staticmethod(lambda model_config: None),
)
event = _native_session_create("sid-native-default-ref", modalities=["text"])
config = DuplexSessionConfig.from_event(event)
runtime_config = await MiniCPMO45NativeDuplexServingAdapter.prepare_runtime_config(
config,
model_config=SimpleNamespace(model="openbmb/MiniCPM-o-4_5"),
)
assert "ref_audio_data" not in runtime_config
assert "ref_audio_format" not in runtime_config
assert "ref_audio_sample_rate_hz" not in runtime_config
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_preserves_ref_audio_channels_until_normalize(monkeypatch):
class FakeMediaConnector:
def __init__(self, **kwargs):
self.kwargs = kwargs
async def fetch_audio_async(self, ref_audio):
assert ref_audio == "data:audio/wav;base64,AAAA"
return np.tile(np.array([[0.25, -0.25], [0.5, -0.5]], dtype=np.float32), (800, 1)), 16000
monkeypatch.setattr(
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.adapter.MediaConnector",
FakeMediaConnector,
)
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
event = _native_session_create("sid-native-ref-audio-stereo", modalities=["text", "audio"])
event["session"]["ref_audio"] = "data:audio/wav;base64,AAAA"
ws = TimedWebSocket()
ws.put(event)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
runtime_config = engine.opened_runtime_configs[0]
assert base64.b64decode(runtime_config["ref_audio_data"]) == struct.pack("<1600f", *([0.0] * 1600))
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_rejects_ref_audio_path():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
event = _native_session_create("sid-native-ref-path")
event["session"]["extra_body"] = {
"native_duplex": True,
"ref_audio_path": "/tmp/ref.wav",
}
ws = TimedWebSocket()
ws.put(event)
await handler.handle_session(ws)
assert ws.sent_types() == ["error"]
assert ws.sent[0]["code"] == "unsupported_ref_audio_path"
assert engine.opened == []
def test_minicpmo_native_duplex_explicit_barge_in_request_interrupts():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-native-explicit-barge-disabled",
config=DuplexSessionConfig(overlap_policy=DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value),
capabilities=minicpmo45_native_capabilities(),
)
payload = {
"type": "audio",
"audio": _pcm_f32_b64(16000, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
"is_speech": True,
}
decision = handler._overlap_decision(
session,
{"force_barge_in": True},
payload,
)
assert decision == {
"action": "barge_in",
"reason": "client_force_barge_in",
"duration_ms": 1000,
"buffer_audio": True,
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"barge_in_event",
[
{"type": "barge_in"},
{"type": "turn.signal", "event": "barge_in"},
],
)
async def test_minicpmo_native_duplex_accepts_explicit_barge_in_control(
barge_in_event: dict[str, object],
):
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-barge-control-disabled"))
ws.put(barge_in_event)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert not any(message.get("code") == "barge_in_unsupported" for message in ws.sent)
assert ("sid-native-barge-control-disabled", "barge_in") in engine.signals
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_explicit_non_speech_stays_listening_without_append():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-silence"))
ws.put(
{
"type": "input_audio_buffer.append",
"audio": "AAAA",
"format": "pcm_f32le",
"is_speech": False,
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert engine.appended == []
assert "response.listen" in ws.sent_types()
assert "response.created" not in ws.sent_types()
assert "response.output_audio.delta" not in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_drains_data_plane_stream_until_done():
def _stage_output(samples: int, *, finished: bool):
return SimpleNamespace(
request_id="duplex-sid-native-stream-e0-stage0-s1",
finished=finished,
outputs=[
SimpleNamespace(
text="",
multimodal_output={
"audio": np.zeros(samples, dtype=np.float32),
"sr": 24000,
},
)
],
)
control_result = {
"operation": "append",
"session_id": "sid-native-stream",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {
"supported": True,
"implementation_level": "model_native_duplex",
"data_plane_append": True,
"request_id": "duplex-sid-native-stream-e0-stage0-s1",
"response_stage_id": 1,
},
}
],
"data_plane_outputs": [_stage_output(10, finished=False)],
}
engine = FakeEngineClient(
append_result=control_result,
collect_outputs=[
[_stage_output(20, finished=False)],
[_stage_output(30, finished=True)],
],
)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
def close_on_second_created(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if ws.sent_types().count("response.created") >= 2:
ws.put({"type": "session.close"})
ws = TimedWebSocket(on_send=close_on_second_created)
ws.put(_native_session_create("sid-native-stream"))
ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"})
await handler.handle_session(ws)
deltas = [m for m in ws.sent if m.get("type") == "response.output_audio.delta"]
assert [m["audio"] for m in deltas] == ["wav-10", "wav-10", "wav-10"]
assert [m["end_of_turn"] for m in deltas] == [False, False, True]
assert len([m for m in ws.sent if m.get("type") == "response.done"]) == 1
assert engine.collected == [
("duplex-sid-native-stream-e0-stage0-s1", 1),
("duplex-sid-native-stream-e0-stage0-s1", 1),
]
@pytest.mark.asyncio
async def test_minicpmo_auto_response_restarts_drain_when_append_races_idle_exit(
monkeypatch: pytest.MonkeyPatch,
):
request_id = "duplex-sid-native-late-output-e0-stage0"
late_output = object()
engine = FakeEngineClient(
collect_outputs=[[], [], [], [late_output]],
collect_delay_s=0.05,
)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
session = DuplexSession(
session_id="sid-native-late-output",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
native = handler._runtime_session_state(session)
projected_batches: list[object] = []
projected = asyncio.Event()
async def project_late_output(_send_json, result, **_kwargs):
projected_batches.append(result)
projected.set()
session.close()
return None, True
monkeypatch.setattr(handler, "_send_native_duplex_events", project_late_output)
result = {
"stage_results": [
{
"result": {
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
}
}
]
}
assert (
await handler._start_native_data_plane_stream_task(
None,
result,
session=session,
expected_epoch=session.epoch,
)
is True
)
first_drain = native.data_plane_task
assert first_drain is not None
await asyncio.sleep(0.01)
assert (
await handler._start_native_data_plane_stream_task(
None,
result,
session=session,
expected_epoch=session.epoch,
)
is False
)
await asyncio.wait_for(projected.wait(), timeout=1)
await asyncio.wait_for(first_drain, timeout=1)
assert len(engine.collected) == 4
assert projected_batches == [{"data_plane_outputs": [late_output]}]
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_ignores_outputs_after_data_plane_turn_done():
request_id = "duplex-sid-native-post-done-e0-stage0-s1"
def _stage_output(samples: int, *, turn_end: bool = False):
return SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[
SimpleNamespace(
text="hello",
multimodal_output={
"audio": np.zeros(samples, dtype=np.float32),
"sr": 24000,
"meta": {
"turn_end": turn_end,
"duplex_turn_id": 0,
"duplex_epoch": 0,
},
},
)
],
)
control_result = {
"operation": "append",
"session_id": "sid-native-post-done",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {
"supported": True,
"implementation_level": "model_native_duplex",
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
},
}
],
"data_plane_outputs": [_stage_output(10)],
}
engine = FakeEngineClient(
append_result=control_result,
collect_outputs=[
[_stage_output(20, turn_end=True)],
[_stage_output(30)],
],
)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
def close_on_done(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") == "response.done":
ws.put({"type": "session.close"})
event = _native_session_create("sid-native-post-done")
event["session"]["extra_body"]["auto_response"] = True
ws = TimedWebSocket(on_send=close_on_done)
ws.put(event)
ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"})
await handler.handle_session(ws)
assert ws.sent_types().count("response.created") == 1
assert ws.sent_types().count("response.done") == 1
deltas = [m for m in ws.sent if m.get("type") == "response.output_audio.delta"]
assert [m["audio"] for m in deltas] == ["wav-10", "wav-10"]
assert engine.collected
assert all(collected == (request_id, 1) for collected in engine.collected)
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_accepts_next_append_after_response_done():
def _stage_output(*, finished: bool):
return SimpleNamespace(
request_id="duplex-sid-native-next-turn-e0-stage0-s1",
finished=finished,
outputs=[
SimpleNamespace(
text="",
multimodal_output={
"audio": np.zeros(10, dtype=np.float32),
"sr": 24000,
},
)
],
)
control_result = {
"operation": "append",
"session_id": "sid-native-next-turn",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {
"supported": True,
"implementation_level": "model_native_duplex",
"data_plane_append": True,
"request_id": "duplex-sid-native-next-turn-e0-stage0-s1",
"response_stage_id": 1,
},
}
],
"data_plane_outputs": [_stage_output(finished=True)],
}
engine = FakeEngineClient(append_result=control_result)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
done_count = 0
def send_next_turn_or_close(ws: TimedWebSocket, data: dict[str, Any]) -> None:
nonlocal done_count
if data.get("type") != "response.done":
return
done_count += 1
if done_count == 1:
ws.put({"type": "input_audio_buffer.append", "audio": "BBBB", "format": "pcm_f32le"})
ws.put({"type": "input_audio_buffer.commit", "final": True})
ws.put({"type": "response.create"})
else:
ws.put({"type": "session.close"})
ws = TimedWebSocket(on_send=send_next_turn_or_close, receive_timeout_s=5)
create = _native_session_create("sid-native-next-turn")
create["session"]["idle_timeout_s"] = 5
ws.put(create)
ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"})
await handler.handle_session(ws)
assert len(engine.appended) >= 2, [
(message.get("type"), message.get("code"), message.get("reason")) for message in ws.sent
]
assert len([m for m in ws.sent if m.get("type") == "response.done"]) == 2
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_commit_finalizes_after_streamed_full_chunk():
control_result = {
"operation": "append",
"session_id": "sid-native-auto-finalize",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [],
}
engine = FakeEngineClient(control_result=control_result)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
event = _native_session_create("sid-native-auto-finalize")
event["session"]["extra_body"]["auto_response"] = True
ws.put(event)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert len(engine.appended) == 1
assert engine.appended[0][3] is False
assert "response.created" not in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_commit_finalizes_with_active_data_plane_stream():
request_id = "duplex-sid-native-auto-active-stream-stage0"
control_result = {
"operation": "append",
"session_id": "sid-native-auto-active-stream",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"result": {
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
}
}
],
}
engine = FakeEngineClient(
control_result=control_result,
collect_outputs=[[], [], []],
collect_delay_s=0.05,
)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
event = _native_session_create("sid-native-auto-active-stream")
event["session"]["extra_body"]["auto_response"] = True
ws.put(event)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert len(engine.appended) == 1
assert engine.appended[0][3] is False
assert "response.created" not in ws.sent_types()
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_accepts_realtime_commit_after_stream_consumed():
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket()
event = _native_session_create("sid-native-auto-consumed-stream")
event["session"]["extra_body"]["auto_response"] = True
ws.put(event)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
ws.put(
{
"type": "input_audio_buffer.commit",
"final": True,
"realtime_item_id": "item-consumed-stream",
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert not any(message.get("code") == "input_audio_buffer_empty" for message in ws.sent)
committed = next(message for message in ws.sent if message.get("type") == "input.committed")
assert committed["realtime_item_id"] == "item-consumed-stream"
assert committed["native_audio"] is True
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_post_response_commit_ignores_stale_progress_state():
session_id = "sid-native-auto-post-response"
engine = FakeEngineClient()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") != "session.created":
return
session = handler._registry.get(session_id)
assert session is not None
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(8000),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
ws.put({"type": "session.close"})
ws = TimedWebSocket(on_send=on_send)
create = _native_session_create(session_id)
create["session"]["extra_body"]["auto_response"] = True
ws.put(create)
await handler.handle_session(ws)
assert len(engine.appended) == 1
assert engine.appended[0][3] is True
assert "input.committed" in ws.sent_types()
assert not any(message.get("code") == "response_already_active" for message in ws.sent)
def test_realtime_overlap_commit_uses_current_generation_identity():
handler, session = _auto_response_context("sid-realtime-reserved-input-turn")
session.turn_id = 1
session.bind_response_turn(1)
session.turn_id = 3
committed = handler._commit_native_audio_input(session)
assert committed.turn_id == 3
assert session.turn_id == 3
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_keeps_request_bound_for_segment_continuation():
request_id = "duplex-sid-native-auto-continuation-e0-stage0"
control_result = {
"operation": "append",
"session_id": "sid-native-auto-continuation",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"result": {
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
}
}
],
}
terminal_segment = _duplex_tts_output(
request_id=request_id,
samples=0,
finished=True,
tts_is_last_chunk=True,
token_ids=[151645],
)
engine = FakeEngineClient(
control_result=control_result,
collect_outputs=[[terminal_segment], [], []],
collect_delay_s=0.05,
)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") != "session.created":
return
session = handler._registry.get("sid-native-auto-continuation")
assert session is not None
session.capabilities.chunk_period_ms = 50
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(800),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
ws.put({"type": "input_audio_buffer.commit", "final": True})
ws = TimedWebSocket(on_send=on_send)
event = _native_session_create("sid-native-auto-continuation")
event["session"]["extra_body"]["auto_response"] = True
ws.put(event)
await handler.handle_session(ws)
assert len(engine.appended) >= 3
_, mode, payload, final = engine.appended[2]
assert mode == "append_audio_chunk"
assert payload["duplex_turn_id"] == 0
assert final is False
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_real_input_preempts_unsent_silence():
request_id = "duplex-sid-native-real-before-silence-e0-stage0"
class SequencedAppendEngine(FakeEngineClient):
def __init__(self) -> None:
control_result = {
"operation": "append",
"session_id": "sid-native-real-before-silence",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"result": {
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
}
}
],
}
terminal_segment = _duplex_tts_output(
request_id=request_id,
samples=0,
finished=True,
tts_is_last_chunk=True,
token_ids=[151645],
)
super().__init__(
control_result=control_result,
collect_outputs=[[terminal_segment], [], []],
collect_delay_s=0.01,
)
self.append_sequence: list[str] = []
self.first_real_started = asyncio.Event()
self.second_real_started = asyncio.Event()
self.real_count = 0
async def append_duplex_input_async(self, session_id: str, **kwargs):
kwargs.pop("expected_epoch", None)
result = await super().append_duplex_input_async(session_id, **kwargs)
payload = kwargs.get("payload")
is_silence = (
isinstance(payload, dict)
and payload.get("audio") == OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO
)
if is_silence:
self.append_sequence.append("silence")
else:
self.real_count += 1
self.append_sequence.append(f"real-{self.real_count}")
if self.real_count == 1:
self.first_real_started.set()
elif self.real_count == 2:
self.second_real_started.set()
return result
session_created = asyncio.Event()
engine = SequencedAppendEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") != "session.created":
return
session = handler._registry.get("sid-native-real-before-silence")
assert session is not None
session.capabilities.chunk_period_ms = 50
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(800),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
session_created.set()
ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0)
event = _native_session_create("sid-native-real-before-silence")
event["session"]["extra_body"]["auto_response"] = True
ws.put(event)
handler_task = asyncio.create_task(handler.handle_session(ws))
await asyncio.wait_for(session_created.wait(), timeout=1)
await asyncio.wait_for(engine.first_real_started.wait(), timeout=1)
await asyncio.sleep(0.02)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(800, value=0.07),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
await asyncio.wait_for(engine.second_real_started.wait(), timeout=1)
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert engine.append_sequence[:2] == ["real-1", "real-2"]
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_silence_tail():
request_id = "duplex-sid-native-silence-tail-e0-stage0"
class SubmittedSilenceEngine(FakeEngineClient):
def __init__(self) -> None:
control_result = {
"operation": "append",
"session_id": "sid-native-silence-tail",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"result": {
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
}
}
],
}
terminal_segment = _duplex_tts_output(
request_id=request_id,
samples=0,
finished=True,
tts_is_last_chunk=True,
token_ids=[151645],
)
super().__init__(
control_result=control_result,
collect_outputs=[[terminal_segment], [], []],
collect_delay_s=0.01,
)
self.silence_append_started = asyncio.Event()
self.release_silence_append = asyncio.Event()
self.append_sequence: list[str] = []
self.real_append_count = 0
async def append_duplex_input_async(self, session_id: str, **kwargs):
kwargs.pop("expected_epoch", None)
payload = kwargs.get("payload")
is_silence_continuation = (
isinstance(payload, dict)
and payload.get("format") == "pcm_f32le"
and payload.get("audio") == OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO
)
if is_silence_continuation:
result = await super().append_duplex_input_async(session_id, **kwargs)
self.append_sequence.append("silence-started")
self.silence_append_started.set()
await self.release_silence_append.wait()
self.append_sequence.append("silence-returned")
return result
result = await super().append_duplex_input_async(session_id, **kwargs)
if (
isinstance(payload, dict)
and payload.get("audio") != OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO
):
self.real_append_count += 1
self.append_sequence.append(f"real-{self.real_append_count}")
return result
engine = SubmittedSilenceEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") != "session.created":
return
session = handler._registry.get("sid-native-silence-tail")
assert session is not None
session.capabilities.chunk_period_ms = 50
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(800, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0)
event = _native_session_create("sid-native-silence-tail")
event["session"]["extra_body"]["auto_response"] = True
ws.put(event)
handler_task = asyncio.create_task(handler.handle_session(ws))
await asyncio.wait_for(engine.silence_append_started.wait(), timeout=1)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000, value=0.07),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
await asyncio.sleep(0.05)
assert engine.real_append_count == 1
assert engine.append_sequence == ["real-1", "silence-started"]
engine.release_silence_append.set()
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert engine.real_append_count == 2
assert engine.append_sequence == ["real-1", "silence-started", "silence-returned", "real-2"]
@pytest.mark.asyncio
async def test_minicpmo_native_auto_response_preserves_silence_continuations_across_append_tails():
request_id = "duplex-sid-native-continuation-tails-e0-stage0"
class SequencedAppendEngine(FakeEngineClient):
def __init__(self) -> None:
control_result = {
"operation": "append",
"session_id": "sid-native-continuation-tails",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"result": {
"data_plane_append": True,
"request_id": request_id,
"response_stage_id": 1,
}
}
],
}
super().__init__(control_result=control_result)
self.real_append_started = asyncio.Event()
self.release_real_append = asyncio.Event()
self.first_silence_started = asyncio.Event()
self.release_first_silence = asyncio.Event()
self.second_silence_started = asyncio.Event()
self.release_collect = asyncio.Event()
self.silence_count = 0
async def append_duplex_input_async(self, session_id: str, **kwargs):
kwargs.pop("expected_epoch", None)
payload = kwargs.get("payload")
assert isinstance(payload, dict)
result = await super().append_duplex_input_async(session_id, **kwargs)
if payload.get("audio") != OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO:
self.real_append_started.set()
await self.release_real_append.wait()
return result
self.silence_count += 1
if self.silence_count == 1:
self.first_silence_started.set()
await self.release_first_silence.wait()
else:
self.second_silence_started.set()
return result
async def collect_duplex_data_plane_outputs_async(
self,
request_id: str,
*,
response_stage_id: int | None = None,
timeout: float | None = None,
) -> list[object]:
del timeout
self.collected.append((request_id, response_stage_id))
await self.release_collect.wait()
return []
session_created = asyncio.Event()
def on_send(_ws: TimedWebSocket, data: dict[str, Any]) -> None:
if data.get("type") == "session.created":
session_created.set()
engine = SequencedAppendEngine()
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
idle_timeout_s=1,
)
ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0)
event = _native_session_create("sid-native-continuation-tails")
event["session"]["extra_body"]["auto_response"] = True
ws.put(event)
ws.put(
{
"type": "input_audio_buffer.append",
"audio": _pcm_f32_b64(16000, value=0.05),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
handler_task = asyncio.create_task(handler.handle_session(ws))
await asyncio.wait_for(session_created.wait(), timeout=1)
await asyncio.wait_for(engine.real_append_started.wait(), timeout=1)
session = handler._registry.get("sid-native-continuation-tails")
assert session is not None
session.capabilities.chunk_period_ms = 50
session.bind_request(request_id)
response_id = session.begin_response(turn_id=session.turn_id)
native = handler._runtime_session_state(session)
scheduler = native.silence_continuation_scheduler
assert scheduler is not None
payload = handler._native_silence_unit_payload()
payload["duplex_turn_id"] = session.turn_id
kwargs = {
"request_id": request_id,
"owner_id": f"response:{response_id}",
"response_id": response_id,
"response_owned": True,
"expected_epoch": session.epoch,
"expected_incarnation": session.incarnation,
"expected_model_turn_id": None,
"send_json": ws.send_json,
}
assert await scheduler(payload, **kwargs) is True
next_schedule = asyncio.create_task(scheduler(payload, **kwargs))
await asyncio.sleep(0.05)
assert not next_schedule.done()
engine.release_real_append.set()
await asyncio.wait_for(engine.first_silence_started.wait(), timeout=1)
assert not next_schedule.done()
engine.release_first_silence.set()
assert await asyncio.wait_for(next_schedule, timeout=1) is True
await asyncio.wait_for(engine.second_silence_started.wait(), timeout=1)
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert engine.silence_count == 2
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_uses_segment_text_metadata_for_transcript_cursor():
request_id = "duplex-sid-native-repeat-text-e0-stage0"
def _stage_output(
text: str,
samples: int,
*,
segment_text: str,
turn_id: int = 0,
turn_end: bool = False,
token_ids: list[int] | None = None,
):
return SimpleNamespace(
request_id=request_id,
finished=False,
outputs=[
SimpleNamespace(
text=text,
token_ids=list(token_ids or []),
multimodal_output={
"audio": np.zeros(samples, dtype=np.float32),
"sr": 24000,
"meta.llm_output_text_utf8": np.frombuffer(segment_text.encode("utf-8"), dtype=np.uint8),
"meta": {"duplex_turn_id": turn_id, "turn_end": turn_end},
},
)
],
)
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-native-repeat-text",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
sent: list[dict[str, Any]] = []
async def send_json(payload: dict[str, Any]) -> None:
sent.append(payload)
async def emit(output: object) -> None:
result = {"data_plane_outputs": [output]}
for native_result in handler._serving_runtime_adapter.data_plane.project(
result,
context=_data_plane_context(session),
):
await handler._send_one_native_duplex_event(send_json, native_result, session=session)
await emit(_stage_output("same reply", 24000, segment_text="same reply"))
await emit(
_stage_output("same replysame reply", 24000, segment_text="same reply", turn_end=True, token_ids=[151645])
)
await emit(_stage_output("same replysame replysame reply", 48000, segment_text="same reply", turn_id=1))
await emit(
_stage_output(
"same replysame replysame replysame reply",
48000,
segment_text="same reply",
turn_id=1,
turn_end=True,
token_ids=[151645],
)
)
created = [m for m in sent if m.get("type") == "response.created"]
done = [m for m in sent if m.get("type") == "response.done"]
deltas = [m for m in sent if m.get("type") == "response.output_audio.delta" and m.get("audio")]
protocol = NativeRealtimeSessionProtocol(TimedWebSocket()) # type: ignore[arg-type]
realtime_events = [event for payload in sent for event in protocol._from_duplex_event(payload)]
transcript_deltas = [
m for m in realtime_events if m.get("type") == "response.audio_transcript.delta" and m.get("delta")
]
assert len(created) == 2
assert len(done) == 2
assert len(deltas) == 2
assert [m["text"] for m in deltas] == ["same reply", "same reply"]
assert [m["audio"] for m in deltas] == ["wav-24000", "wav-24000"]
assert [m["delta"] for m in transcript_deltas] == ["same reply", "same reply"]
assert [m["response_id"] for m in deltas] == [m["response_id"] for m in created]
assert [m["response_id"] for m in done] == [m["response_id"] for m in created]
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_continuous_speak_reuses_active_response_until_turn_end():
request_id = "duplex-sid-native-continuous-speak-e0-stage0"
def _stage_output(
text: str,
samples: int,
*,
turn_end: bool = False,
token_ids: list[int] | None = None,
):
return SimpleNamespace(
request_id=request_id,
finished=turn_end,
outputs=[
SimpleNamespace(
text=text,
token_ids=list(token_ids or []),
multimodal_output={},
)
],
multimodal_output={
"audio": np.zeros(samples, dtype=np.float32),
"sr": 24000,
"meta.duplex_turn_id": np.array([0], dtype=np.int32),
"meta.duplex_epoch": np.array([0], dtype=np.int32),
"meta.turn_end": np.array([int(turn_end)], dtype=np.int32),
"meta.tts_is_last_chunk": np.array([int(turn_end)], dtype=np.int32),
},
)
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-native-continuous-speak",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
sent: list[dict[str, Any]] = []
async def send_json(payload: dict[str, Any]) -> None:
sent.append(payload)
async def emit(output: object) -> None:
result = {"data_plane_outputs": [output]}
for native_result in handler._serving_runtime_adapter.data_plane.project(
result,
context=_data_plane_context(session),
):
await handler._send_one_native_duplex_event(send_json, native_result, session=session)
await emit(_stage_output("hello", 24000))
await emit(_stage_output("hello again", 48000))
created = [m for m in sent if m.get("type") == "response.created"]
deltas = [m for m in sent if m.get("type") == "response.output_audio.delta" and m.get("audio")]
assert len(created) == 1
assert len(deltas) == 2
assert {m["response_id"] for m in deltas} == {created[0]["response_id"]}
assert "response.done" not in [m.get("type") for m in sent]
await emit(_stage_output("hello again", 48000, turn_end=True, token_ids=[151645]))
done = [m for m in sent if m.get("type") == "response.done"]
assert len(done) == 1
assert done[0]["response_id"] == created[0]["response_id"]
assert session.active_response_id is None
assert session.turn_id == 1
@pytest.mark.asyncio
async def test_continuous_response_metrics_accumulate_only_owned_model_units():
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-response-metrics",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
sent: list[dict[str, Any]] = []
async def send_json(payload: dict[str, Any]) -> None:
sent.append(payload)
async def emit(*, text: str, audio: str, tokens: int, ttft_ms: float, itls: list[float]) -> None:
await handler._send_one_native_duplex_event(
send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": "duplex-sid-response-metrics-e0-stage0",
"text": text,
"audio_data": audio,
"audio_format": "pcm16",
"audio_duration_ms": 100,
"end_of_turn": False,
"model_turn_id": 0,
"stage_metrics": {
"0": {
"num_tokens_out": tokens,
"stage_gen_time_ms": 80.0,
"vllm_ttft_ms": ttft_ms,
"vllm_tpot_ms": 9.0,
"vllm_itls_ms": itls,
}
},
},
session=session,
)
await emit(text="first", audio="audio-a", tokens=4, ttft_ms=50.0, itls=[10.0, 11.0])
await emit(text="second", audio="audio-b", tokens=6, ttft_ms=70.0, itls=[12.0, 13.0, 14.0])
deltas = [payload for payload in sent if payload.get("type") == "response.output_audio.delta"]
assert len(deltas) == 2
first_metrics = deltas[0]["vllm_omni"]["stage_metrics"]["0"]
second_metrics = deltas[1]["vllm_omni"]["stage_metrics"]["0"]
assert first_metrics["num_tokens_out"] == 4
assert second_metrics["num_tokens_out"] == 10
assert second_metrics["vllm_ttft_ms"] == 50.0
assert second_metrics["vllm_itls_ms"] == [10.0, 11.0, 12.0, 13.0, 14.0]
first_response_id = deltas[0]["response_id"]
session.end_response()
await emit(text="third", audio="audio-c", tokens=3, ttft_ms=90.0, itls=[15.0, 16.0])
third_delta = [payload for payload in sent if payload.get("type") == "response.output_audio.delta"][-1]
third_metrics = third_delta["vllm_omni"]["stage_metrics"]["0"]
assert third_delta["response_id"] != first_response_id
assert third_metrics["num_tokens_out"] == 3
assert third_metrics["vllm_ttft_ms"] == 90.0
assert third_metrics["vllm_itls_ms"] == [15.0, 16.0]
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_drops_old_turn_audio_while_new_response_is_active():
request_id = "duplex-sid-native-old-turn-audio-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-native-old-turn-audio",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
response_id = session.begin_response(turn_id=1)
session.turn_id = 2
session.bind_request(request_id)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"text": "stale",
"audio_data": "old-audio",
"audio_format": "pcm16",
"audio_duration_ms": 100,
"end_of_turn": False,
"model_turn_id": 0,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is False
assert session.active_response_id == response_id
assert ws.sent == []
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_future_turn_audio_starts_next_response():
request_id = "duplex-sid-native-future-turn-audio-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-native-future-turn-audio",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
first_response_id = session.begin_response(turn_id=0)
session.bind_request(request_id)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "tts",
"is_listen": False,
"data_plane_request_id": request_id,
"text": "next",
"audio_data": "next-audio",
"audio_format": "pcm16",
"audio_duration_ms": 100,
"end_of_turn": False,
"model_turn_id": 1,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is True
sent_types = [event["type"] for event in ws.sent]
assert sent_types == [
"response.done",
"response.created",
"response.speak",
"response.output_audio.delta",
]
assert ws.sent[0]["response_id"] == first_response_id
assert ws.sent[1]["response_id"] != first_response_id
assert ws.sent[3]["response_id"] == ws.sent[1]["response_id"]
assert ws.sent[3]["vllm_omni"]["model_turn_id"] == 1
assert session.active_response_turn_id == 1
assert session.turn_id == 1
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_drops_old_turn_listen_while_new_response_is_active():
request_id = "duplex-sid-native-old-turn-listen-e0-stage0"
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
session = DuplexSession(
session_id="sid-native-old-turn-listen",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
response_id = session.begin_response(turn_id=1)
session.turn_id = 2
session.bind_request(request_id)
ws = TimedWebSocket()
close_reason, emitted = await handler._send_one_native_duplex_event(
ws.send_json,
{
"supported": True,
"stage_role": "llm",
"is_listen": True,
"model_listen": True,
"data_plane_request_id": request_id,
"end_of_turn": False,
"model_turn_id": 0,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
},
session=session,
expected_epoch=session.epoch,
)
assert close_reason is None
assert emitted is False
assert session.active_response_id == response_id
assert ws.sent == []
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_silence_continuation_requires_active_request():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
session = DuplexSession(
session_id="sid-native-stale-continuation",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.begin_response()
session.bind_request("duplex-sid-native-stale-continuation-stage0")
session.mark_audio_sent(100)
sent: list[dict[str, Any]] = []
async def send_json(data: dict[str, Any]) -> None:
sent.append(data)
_install_direct_silence_scheduler(handler, session)
await handler._maybe_continue_native_response(send_json, session=session, expected_epoch=session.epoch)
session.clear_request()
await asyncio.sleep(0.01)
assert engine.appended == []
@pytest.mark.asyncio
async def test_minicpmo_auto_response_segment_complete_continues_until_model_listen():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
session = DuplexSession(
session_id="sid-native-segment-complete",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response()
request_id = "duplex-sid-native-segment-complete-stage0"
session.bind_request(request_id)
session.mark_audio_sent(1000)
_install_direct_silence_scheduler(handler, session)
sent: list[dict[str, Any]] = []
async def send_json(data: dict[str, Any]) -> None:
sent.append(data)
close_reason, emitted = await handler._send_one_native_duplex_event(
send_json,
{
"supported": True,
"stage_role": "llm",
"is_listen": True,
"model_listen": False,
"listen_source": "auto_response_segment_complete",
"reason": "auto_response_segment_complete",
"data_plane_request_id": request_id,
"end_of_turn": False,
"uses_model_runner_scheduler": True,
"runner_kv_backed": True,
"runtime_impl": "scheduler_data_plane",
"owned_runtime": False,
},
session=session,
expected_epoch=session.epoch,
)
await asyncio.sleep(0.01)
assert close_reason is None
assert emitted is False
assert session.active_response_id == response_id
assert session.active_request_id == request_id
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
assert sent == []
assert len(engine.appended) == 1
_, mode, payload, final = engine.appended[0]
assert mode == "append_audio_chunk"
assert payload["type"] == "audio"
assert payload.get("force_listen") is not True
assert final is False
@pytest.mark.parametrize(
("steps", "expected_deltas"),
[
(
[("你好呀", 1), ("你好呀", 1), ("你好呀", 2)],
["你好呀", "", "你好呀"],
),
(
[("你好呀", 1), ("你好呀", 2)],
["你好呀", "你好呀"],
),
],
ids=["turn-scoped", "exact-repeated-turn-text"],
)
def test_minicpmo_data_plane_text_cursor(
steps: list[tuple[str, int]],
expected_deltas: list[str],
):
data_plane = _test_data_plane()
request_id = "duplex-sid-native-e0-stage0"
deltas = [data_plane.segment_text_delta(request_id, text, turn_id=turn_id) for text, turn_id in steps]
assert deltas == expected_deltas
def test_minicpmo_data_plane_text_cursor_does_not_clip_prior_turn_overlap():
data_plane = _test_data_plane()
request_id = "duplex-sid-native-e0-stage0"
assert data_plane.segment_text_delta(request_id, "你好呀。你有什么想聊的吗?", turn_id=1)
assert (
data_plane.segment_text_delta(request_id, "你有什么想聊的吗?你好呀。", turn_id=2)
== "你有什么想聊的吗?你好呀。"
)
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_cancel_interrupts_background_data_plane_stream():
control_result = {
"operation": "append",
"session_id": "sid-native-cancel-stream",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [
{
"stage_id": 0,
"replica_id": 0,
"result": {
"supported": True,
"implementation_level": "model_native_duplex",
"data_plane_append": True,
"request_id": "duplex-sid-native-cancel-stream-e0-stage0-s1",
"response_stage_id": 1,
},
}
],
}
engine = FakeEngineClient(
append_result=control_result,
collect_delay_s=0.2,
collect_outputs=[
[
SimpleNamespace(
request_id="duplex-sid-native-cancel-stream-e0-stage0-s1",
finished=True,
outputs=[
SimpleNamespace(
text="",
multimodal_output={"audio": np.zeros(10, dtype=np.float32), "sr": 24000},
)
],
)
]
],
)
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-cancel-stream"))
ws.put({"type": "input_audio_buffer.append", "audio": "AAAA", "format": "pcm_f32le"})
ws.put({"type": "input.cancel"})
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert "audio.cancelled" in ws.sent_types()
assert "response.output_audio.delta" not in ws.sent_types()
assert engine.signals == [("sid-native-cancel-stream", "barge_in")]
assert engine.signal_fences == [DuplexFence("sid-native-cancel-stream")]
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_session_close_discards_partial_pcm_tail():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-short-chunks"))
chunk = _pcm_f32_b64(3200)
for _ in range(6):
ws.put(
{
"type": "input_audio_buffer.append",
"audio": chunk,
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
)
ws.put({"type": "session.close"})
await handler.handle_session(ws)
assert len(engine.appended) == 1
_, mode, payload, final = engine.appended[0]
assert mode == "append_audio_chunk"
assert final is False
assert isinstance(payload, dict)
assert len(base64.b64decode(payload["audio"])) == 16000 * 4
assert "duplex_num_input_tokens" not in payload
@pytest.mark.asyncio
async def test_minicpmo_native_duplex_idle_timeout_closes_runtime_with_timeout_reason():
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=0.1)
ws = TimedWebSocket()
ws.put(_native_session_create("sid-native-timeout"))
await handler.handle_session(ws)
assert ws.sent_types() == ["session.created", "session.closed"]
assert ws.sent[-1]["reason"] == "timeout"
assert engine.closed == [("sid-native-timeout", "timeout")]