[Core][Frontend] Unified Full-duplex Framework (#7413)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
chickeytonandClaude Opus 5 authored and GitHub committed 2026-09-16 15:35:36 +08:00
1 parent 9005d78903
commit 99ff4f3070
182 files changed
+26951 -34743

No files matched your search

+41 -27
View File
@@ -135,6 +135,7 @@ class MockResponse:
async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
class FakeRealtimeClient:
last_instance = None
instances: list = []
def __init__(self, url):
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
@@ -143,6 +144,9 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
self.sent = []
self.response_count = 0
self.ack_count = 0
self.silence_seconds = []
self.closed = 0
type(self).instances.append(self)
type(self).last_instance = self
async def __aenter__(self):
@@ -155,10 +159,22 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
assert model == "openbmb/MiniCPM-o-4_5"
self.configure_kwargs = kwargs
async def stream_silence(self, *, seconds, chunk_ms=200, until=None):
# A model-native session speaks off its seeded context once audio
# units arrive; the silence itself carries no content. The probe
# stops the silence as soon as the turn settles.
self.silence_seconds.append(seconds)
self._emit_response()
assert until is not None and until()
return 1.0
async def send(self, event):
self.sent.append(event)
if event["type"] != "response.create":
return
self._emit_response()
def _emit_response(self):
self.response_count += 1
response_id = f"resp-{self.response_count}"
now = time.monotonic()
@@ -210,7 +226,7 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
self.ack_count += 1
async def close_session(self, **_kwargs):
return None
self.closed += 1
monkeypatch.setattr(
"vllm_omni.benchmarks.patch.patch._RealtimeTTSProbe",
@@ -239,30 +255,27 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
session=None,
)
client = FakeRealtimeClient.last_instance
assert client.configure_kwargs["native_duplex"] is False
assert client.configure_kwargs["extra_body"] == {
"ref_audio": "data:audio/wav;base64,AAAA",
"return_stage_metrics": True,
}
assert client.sent == [
event
for index in range(4)
for event in (
{
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": f"text {index}"}],
},
},
{"type": "response.create"},
)
]
assert client.ack_count == 4
# One session per utterance: a model-native duplex session takes its text
# once, in the session context, so it cannot be re-seeded for a second one.
sessions = FakeRealtimeClient.instances
assert len(sessions) == 4
for index, client in enumerate(sessions):
assert "native_duplex" not in client.configure_kwargs
assert client.configure_kwargs["extra_body"] == {
"ref_audio": "data:audio/wav;base64,AAAA",
"return_stage_metrics": True,
"duplex_initial_user_text": f"text {index}",
"force_listen_count": 0,
}
# The target text rides the session context, never a conversation item:
# a text-only response.create is rejected by a model-native session.
assert client.sent == []
assert client.silence_seconds and all(seconds > 0 for seconds in client.silence_seconds)
assert client.ack_count == 1
assert client.closed == 1
assert output.success is True
assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
assert output.generated_text == "turn 1 turn 1 turn 1 turn 1"
assert output.audio_duration == pytest.approx(0.4)
assert output.ttft > 0
assert output.audio_ttfp > output.ttft
@@ -281,11 +294,12 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
"session_id": session_id,
"request_index": index,
"utterance_id": f"utt-{index}",
"response_id": f"resp-{index + 1}",
# Response ids are per session, and each utterance now gets its own.
"response_id": "resp-1",
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "conversation.item.create client send to first non-empty text delta",
"ttfp": "conversation.item.create client send to first audio packet",
"ttft": "first silence append client send to first non-empty text delta",
"ttfp": "first silence append client send to first audio packet",
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
"tpot": "Stage-0 engine mean time per output token",
},
+24 -3
View File
@@ -467,6 +467,27 @@ def test_tolerated_playback_ack_rejection_is_a_warning_not_a_failure():
oi._raise_if_session_terminated(collector, 0, warnings=warnings)
def test_our_own_close_is_expected_even_though_the_server_stamps_a_reason():
"""``session.close`` is answered with ``session.closed`` carrying ``client_close``.
The guard exists to catch a session that ended for a reason we did not ask
for. A close we requested is not that, whether or not the server names it.
"""
collector = _collector(({"type": "session.closed", "reason": "client_close"}, 1.0))
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
collector = _collector(({"type": "session.closed", "event": {"reason": "client_close"}}, 1.0))
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
collector = _collector(({"type": "session.closed", "reason": "disconnect"}, 1.0))
with pytest.raises(RuntimeError, match="Unexpected session.closed: disconnect"):
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
collector = _collector(({"type": "session.expired", "reason": "timeout"}, 1.0))
with pytest.raises(RuntimeError, match="session.expired: timeout"):
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
@pytest.mark.parametrize(
("event", "match"),
[
@@ -561,8 +582,8 @@ class _RealtimeClient(oi._RealtimeSession):
instances: list[_RealtimeClient] = []
def __init__(self, config: oi.OmniInteractBenchmarkConfig, session_id: str, reference_audio: str):
super().__init__(config, session_id, reference_audio)
def __init__(self, config: oi.OmniInteractBenchmarkConfig, reference_audio: str):
super().__init__(config, reference_audio)
self.acks: list[tuple[str, int]] = []
self.instances.append(self)
@@ -621,7 +642,7 @@ async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monke
assert "autostart=0" in _RealtimeClient.instances[-1].url
session_config = _RealtimeClient.instances[-1].session_config
assert session_config.extra_body["custom"] == "value"
assert session_config.extra_body["native_duplex"] is True
assert "native_duplex" not in session_config.extra_body
assert session_config.ref_audio == "data:audio/wav;base64,ref"
acks = _RealtimeClient.instances[-1].acks
# Cumulative acks for the one response: an optional 0 ms checkpoint the
+179
View File
@@ -0,0 +1,179 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The Seed-TTS Realtime probe queues for a duplex session slot instead of failing."""
from __future__ import annotations
import pytest
from vllm_omni.benchmarks.patch import patch as bench_patch
from vllm_omni.clients import duplex as duplex_client
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _AdmissionGate:
"""Stands in for ``DuplexClient``: refuses admission ``refusals`` times, then admits."""
instances: list[_AdmissionGate] = []
refusals = 0
refusal_code = "resource_exhausted"
def __init__(self, url: str, **kwargs: object) -> None:
del url, kwargs
self.entered = False
type(self).instances.append(self)
async def __aenter__(self) -> _AdmissionGate:
cls = type(self)
if cls.refusals > 0:
cls.refusals -= 1
raise duplex_client.DuplexProtocolError("duplex_session_capacity_exhausted: limit=4", code=cls.refusal_code)
self.entered = True
return self
async def __aexit__(self, exc_type, exc, tb) -> None:
del exc_type, exc, tb
async def events(self): # the collector's consume loop ends at once
return
yield # pragma: no cover
@pytest.fixture
def gate(monkeypatch: pytest.MonkeyPatch) -> type[_AdmissionGate]:
_AdmissionGate.instances = []
_AdmissionGate.refusals = 0
_AdmissionGate.refusal_code = "resource_exhausted"
monkeypatch.setattr(duplex_client, "DuplexClient", _AdmissionGate)
return _AdmissionGate
async def _configure(probe: bench_patch._RealtimeTTSProbe) -> None:
try:
await probe.configure("test-model", auto_response=True)
finally:
await probe.__aexit__(None, None, None)
@pytest.mark.asyncio
async def test_configure_waits_for_a_free_session_slot(gate) -> None:
"""Two refusals, then a slot: the request is served, not counted as failed."""
gate.refusals = 2
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
await _configure(probe)
assert len(gate.instances) == 3
assert gate.instances[-1].entered is True
assert probe._client is gate.instances[-1]
@pytest.mark.asyncio
async def test_configure_raises_other_rejections_at_once(gate) -> None:
gate.refusals = 1
gate.refusal_code = "unsupported_turn_detection"
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
with pytest.raises(duplex_client.DuplexProtocolError):
await _configure(probe)
assert len(gate.instances) == 1
@pytest.mark.asyncio
async def test_configure_gives_up_when_the_slot_wait_budget_is_spent(gate, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(bench_patch._RealtimeTTSProbe, "_SESSION_SLOT_WAIT_S", 0.0)
gate.refusals = 5
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
with pytest.raises(duplex_client.DuplexProtocolError) as excinfo:
await _configure(probe)
assert excinfo.value.code == "resource_exhausted"
assert len(gate.instances) == 1
class _SilenceSink:
"""Stands in for a configured ``DuplexClient``: records every appended chunk."""
def __init__(self) -> None:
self.config = duplex_client.SessionConfig()
self.chunks: list[bytes] = []
async def append_audio(self, pcm: bytes, *, is_speech: bool | None = None, video_frames=None) -> None:
del video_frames
assert is_speech is False
self.chunks.append(pcm)
@pytest.mark.asyncio
async def test_stream_silence_stops_at_the_turn_end(monkeypatch: pytest.MonkeyPatch) -> None:
"""Silence is fed until the turn settles, not for the whole cap."""
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
sink = _SilenceSink()
probe._client = sink
async def no_wait(_: float) -> None:
return
monkeypatch.setattr(bench_patch.asyncio, "sleep", no_wait)
streamed = await probe.stream_silence(seconds=12.0, chunk_ms=200, until=lambda: len(sink.chunks) >= 3)
assert len(sink.chunks) == 3
assert streamed == pytest.approx(0.6)
assert all(len(chunk) == sink.config.input_audio.byte_count(200) for chunk in sink.chunks)
@pytest.mark.asyncio
async def test_stream_silence_runs_to_the_cap_when_the_turn_never_settles(monkeypatch: pytest.MonkeyPatch) -> None:
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
sink = _SilenceSink()
probe._client = sink
async def no_wait(_: float) -> None:
return
monkeypatch.setattr(bench_patch.asyncio, "sleep", no_wait)
streamed = await probe.stream_silence(seconds=1.0, chunk_ms=200, until=lambda: False)
assert len(sink.chunks) == 5
assert streamed == pytest.approx(1.0)
class _TurnEvents:
def __init__(self, response_ids: list[str], audio: dict[str, bytes], text: dict[str, str] | None = None) -> None:
self.response_ids = response_ids
self._audio = audio
self._text = text or {}
def audio_bytes(self, response_id: str) -> bytes:
return self._audio.get(response_id, b"")
def response_text(self, response_id: str) -> str:
return self._text.get(response_id, "")
def test_turn_response_is_the_first_one_with_audio_even_if_the_model_spoke_again(caplog) -> None:
events = _TurnEvents(["r0", "r1", "r2"], {"r1": b"\x01\x00", "r2": b"\x02\x00"})
with caplog.at_level("WARNING", logger=bench_patch.logger.name):
assert bench_patch._seed_tts_turn_response_id(events, 0, 0) == "r1"
assert "spoke again" in caplog.text
def test_turn_response_ignores_responses_from_before_the_turn() -> None:
events = _TurnEvents(["r0", "r1"], {"r0": b"\x01\x00", "r1": b"\x02\x00"})
assert bench_patch._seed_tts_turn_response_id(events, 1, 0) == "r1"
def test_turn_without_audio_is_an_error() -> None:
events = _TurnEvents(["r0"], {})
with pytest.raises(RuntimeError, match="no audio response"):
bench_patch._seed_tts_turn_response_id(events, 0, 0)
def test_stall_report_says_the_model_never_answered() -> None:
events = _TurnEvents(["r0"], {"r0": b"\x01\x00"})
report = bench_patch._seed_tts_turn_stall_report(events, 1, 3, 30.0)
assert "turn 3 never started a response" in report
assert "30.0s of silence" in report
def test_stall_report_lists_a_response_that_never_finished() -> None:
events = _TurnEvents(["r1"], {"r1": b"\x01\x00\x02\x00"}, {"r1": "hello"})
report = bench_patch._seed_tts_turn_stall_report(events, 0, 0, 4.2)
assert "started 1 response(s) after 4.2s of silence" in report
assert "r1 (4 audio bytes, text 'hello')" in report