[Core] Graduate MiniCPM-o 4.5 and PersonaPlex full-duplex serving out of experimental (#6196)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-05 08:20:21 +00:00
1 parent d4096e01fe
commit 5d7fdf9350
179 files changed
+6462 -3832

No files matched your search

+3 -3
View File
@@ -28,7 +28,7 @@ from vllm_omni.benchmarks.patch.patch import (
async_request_openai_realtime_duplex,
should_request_stage_metrics,
)
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
from vllm_omni.clients.duplex import EventCollector
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
@@ -131,7 +131,7 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
def __init__(self, url):
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
self.events = RealtimeEventCollector()
self.events = EventCollector()
self.configure_kwargs = None
self.sent = []
self.response_count = 0
@@ -206,7 +206,7 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
return None
monkeypatch.setattr(
"vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient",
"vllm_omni.benchmarks.patch.patch._RealtimeTTSProbe",
FakeRealtimeClient,
)
request_input = RequestFuncInput(
+96 -22
View File
@@ -25,9 +25,9 @@ from vllm_omni.benchmarks import serve as benchmark_serve
from vllm_omni.benchmarks.data_modules import omniinteract_dataset as data
from vllm_omni.benchmarks.metrics.metrics import calculate_metrics
from vllm_omni.benchmarks.patch import patch as benchmark_patch
from vllm_omni.clients.duplex import EventCollector
from vllm_omni.entrypoints.cli.benchmark.cli_args import preprocess_serve_args
from vllm_omni.entrypoints.cli.benchmark.serve import OmniBenchmarkServingSubcommand
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
@@ -42,8 +42,8 @@ def _case(tmp_path: Path, *, subset: str = "1q1a", name: str = "video.mp4") -> d
return data.OmniInteractCase(subset, name, video, annotation, "multi_turn")
def _collector(*events: tuple[dict[str, object], float]) -> RealtimeEventCollector:
collector = RealtimeEventCollector()
def _collector(*events: tuple[dict[str, object], float]) -> EventCollector:
collector = EventCollector()
for event, received_at in events:
collector.add(event, received_at_s=received_at)
return collector
@@ -136,7 +136,7 @@ def _successful_output(
*,
audio_time: float = 10.0,
status: str = "completed",
) -> tuple[data.OmniInteractCase, RealtimeEventCollector, oi.OmniInteractCaseResult]:
) -> tuple[data.OmniInteractCase, EventCollector, oi.OmniInteractCaseResult]:
case = _case(tmp_path)
collector = _collector(
(_created(), 9.9),
@@ -151,7 +151,7 @@ def _successful_output(
def _write_success(
root: Path,
case: data.OmniInteractCase,
collector: RealtimeEventCollector,
collector: EventCollector,
result: oi.OmniInteractCaseResult,
**kwargs,
):
@@ -285,7 +285,7 @@ def test_response_ledger_rejects_identity_errors(events, match: str):
class _CompletionClient:
def __init__(self, collector: RealtimeEventCollector):
def __init__(self, collector: EventCollector):
self.events, self.acks = collector, []
def raise_if_reader_stopped(self) -> None:
@@ -334,13 +334,71 @@ async def test_deferred_commit_ignores_the_prior_response_terminal():
@pytest.mark.asyncio
async def test_playback_acks_only_after_serial_audio_drain():
collector = _collector((_audio(samples=2400), 10.0), (_done(), 10.01))
async def test_playback_acks_incrementally_along_the_serial_clock():
# 1 s of audio playing from t=10.0: the first ack checkpoints the
# response as soon as audio arrives, later acks report cumulative
# progress at least every _ACK_STEP_MS of new audio, and the completion
# ack lands exactly at the total once the serial clock drains it.
collector = _collector((_audio(samples=24_000), 10.0), (_done(), 10.01))
client, playback = _CompletionClient(collector), oi._Playback()
await playback.acknowledge(client, now=10.05)
assert client.acks == []
assert client.acks == [("r1", 50)] # checkpoint ack, reserves the slot
await playback.acknowledge(client, now=10.1)
assert client.acks == [("r1", 100)]
assert client.acks == [("r1", 50)] # step not reached, no spam
await playback.acknowledge(client, now=10.3)
assert client.acks == [("r1", 50), ("r1", 300)]
await playback.acknowledge(client, now=11.0)
assert client.acks == [("r1", 50), ("r1", 300), ("r1", 1000)]
assert playback.completion_acked == {"r1"}
await playback.acknowledge(client, now=11.5)
assert client.acks[-1] == ("r1", 1000) # completion acked once
@pytest.mark.asyncio
async def test_playback_checkpoint_ack_fires_before_audio_plays():
# Audio queued but none played yet: a 0 ms checkpoint ack still goes out
# so a later input commit cannot displace the response's history slot.
collector = _collector((_audio(samples=24_000), 10.0), (_done(), 10.01))
client, playback = _CompletionClient(collector), oi._Playback()
await playback.acknowledge(client, now=10.0)
assert client.acks == [("r1", 0)]
@pytest.mark.asyncio
async def test_acks_continue_after_committed_input():
# A later committed user input must not stop the acks: the checkpointed
# response's history slot is updated in place, so the drain ack for the
# tail is still delivered afterwards.
collector = _collector(
(_audio(samples=24_000), 10.0),
(_done(), 10.01),
({"type": "input_audio_buffer.committed"}, 10.4),
)
client, playback = _CompletionClient(collector), oi._Playback()
await playback.acknowledge(client, now=10.05)
assert client.acks == [("r1", 50)]
await playback.acknowledge(client, now=11.5)
assert client.acks == [("r1", 50), ("r1", 1000)]
assert playback.completion_acked == {"r1"}
def test_tolerated_playback_ack_rejection_is_a_warning_not_a_failure():
collector = _collector(
(
{
"type": "error",
"error": {"type": "invalid_request_error", "code": "playback_ack_too_late"},
},
1.0,
),
)
warnings: list[str] = []
oi._raise_if_session_terminated(collector, 0, warnings=warnings)
assert warnings == ["tolerated in-flight server rejection: playback_ack_too_late"]
collector.add({"type": "error", "error": {"code": "bad_event", "message": "boom"}}, received_at_s=1.1)
with pytest.raises(RuntimeError, match="bad_event"):
oi._raise_if_session_terminated(collector, 0, warnings=warnings)
@pytest.mark.parametrize(
@@ -421,30 +479,34 @@ def test_publication_failure_removes_partial_success_bundle(tmp_path: Path, monk
def test_atomic_write_preserves_destination_on_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
destination = tmp_path / "output.wav"
destination.write_bytes(b"old")
monkeypatch.setattr(oi, "write_pcm16_wav", lambda *a, **k: (_ for _ in ()).throw(OSError("write")))
monkeypatch.setattr(
"vllm_omni.clients.duplex.write_pcm16_wav",
lambda *a, **k: (_ for _ in ()).throw(OSError("write")),
)
with pytest.raises(OSError):
oi._atomic_write_wav(destination, b"\0\0", 24_000)
assert destination.read_bytes() == b"old"
assert not list(tmp_path.glob(".*.tmp"))
class _RealtimeClient:
class _RealtimeClient(oi._RealtimeSession):
"""Fake transport over the real session wrapper: URL and session-config
construction stay real, only the network-facing calls are stubbed."""
instances: list[_RealtimeClient] = []
def __init__(self, url: str, **kwargs):
self.url, self.events, self.acks, self.configure_kwargs = url, RealtimeEventCollector(), [], {}
def __init__(self, config: oi.OmniInteractBenchmarkConfig, session_id: str, reference_audio: str):
super().__init__(config, session_id, reference_audio)
self.acks: list[tuple[str, int]] = []
self.instances.append(self)
async def __aenter__(self):
self.events.add({"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 1000}}})
return self
async def __aexit__(self, *args):
return None
async def configure(self, model: str, **kwargs) -> None:
self.configure_kwargs = kwargs
self.events.add({"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 1000}}})
async def send(self, event: dict[str, object]) -> None:
return None
@@ -480,7 +542,7 @@ async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monke
extra_body={"custom": "value"},
)
prepared = data.OmniInteractPreparedInput(1.0, b"\0\0\0\0", ("frame",), "data:audio/wav;base64,ref")
monkeypatch.setattr(oi, "RealtimeDuplexClient", _RealtimeClient)
monkeypatch.setattr(oi, "_RealtimeSession", _RealtimeClient)
monkeypatch.setattr(oi, "_COMPLETION_SETTLE_S", 0.0)
result = await oi.run_omniinteract_case(
case,
@@ -491,8 +553,15 @@ async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monke
assert result.success and result.transcript == "answer" and result._artifact_context is not None
assert not config.output_root.exists()
assert "autostart=0" in _RealtimeClient.instances[-1].url
assert _RealtimeClient.instances[-1].configure_kwargs["extra_body"] == {"custom": "value"}
assert _RealtimeClient.instances[-1].acks == [("r1", 1)]
session_config = _RealtimeClient.instances[-1].session_config
assert session_config.extra_body["custom"] == "value"
assert session_config.extra_body["native_duplex"] is True
assert session_config.ref_audio == "data:audio/wav;base64,ref"
acks = _RealtimeClient.instances[-1].acks
# Cumulative acks for the one response: an optional 0 ms checkpoint the
# moment its audio arrived, then the exact total once drained.
assert acks[-1] == ("r1", 1)
assert acks in ([("r1", 1)], [("r1", 0), ("r1", 1)])
def test_standard_sample_loading_prepares_media_before_timing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
@@ -521,7 +590,12 @@ def test_standard_sample_loading_prepares_media_before_timing(tmp_path: Path, mo
)
preprocess_serve_args(args)
assert args.endpoint == "/v1/realtime"
monkeypatch.setattr(benchmark_patch, "reference_audio_data_url", lambda _: "data:audio/wav;base64,ref")
# The benchmark patch module imports the duplex client library lazily
# (it must stay out of the CLI import graph), so patch the provider.
monkeypatch.setattr(
"vllm_omni.clients.duplex.reference_audio_data_url",
lambda _: "data:audio/wav;base64,ref",
)
monkeypatch.setattr(benchmark_patch, "prepare_media", lambda *a, **k: (1.0, b"pcm", ["frame"]))
samples = benchmark_patch.get_samples(args, None)
assert len(samples) == args.num_prompts == 3
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+895
View File
@@ -0,0 +1,895 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the public duplex client (fake transport, no server)."""
from __future__ import annotations
import asyncio
import base64
import json
import wave
from urllib.parse import parse_qs, urlsplit
import pytest
from vllm_omni.clients.duplex import (
AudioFormat,
ConnectionResumed,
DuplexClient,
DuplexProtocolError,
DuplexSessionClosedError,
EventCollector,
ReconnectPolicy,
SessionConfig,
SessionResumed,
build_realtime_url,
chunk_period_ms,
duplex_unit_boundary_ms,
has_residual_model_unit,
read_pcm16_wav,
reference_audio_data_url,
summarize_session_request_metrics,
write_pcm16_wav,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
SESSION_CREATED = {
"type": "session.created",
"session": {"session_id": "sess-1"},
"incarnation": 0,
"resume_token": "tok-1",
"server_event_seq": 1,
}
SESSION_CLOSED = {"type": "session.closed", "session_id": "sess-1", "server_event_seq": 99}
class FakeSocket:
def __init__(self) -> None:
self.sent: list[dict[str, object]] = []
self.incoming: asyncio.Queue = asyncio.Queue()
self.closed = False
async def send(self, raw: str) -> None:
if self.closed:
raise RuntimeError("socket closed")
self.sent.append(json.loads(raw))
async def recv(self) -> str:
item = await self.incoming.get()
if isinstance(item, BaseException):
raise item
return json.dumps(item)
async def close(self) -> None:
self.closed = True
def feed(self, event: dict[str, object] | BaseException) -> None:
self.incoming.put_nowait(event)
def sent_types(self) -> list[object]:
return [event.get("type") for event in self.sent]
def make_client(*sockets: FakeSocket, **kwargs) -> tuple[DuplexClient, list[str]]:
remaining = list(sockets)
calls: list[str] = []
async def connect(url: str):
calls.append(url)
if not remaining:
raise ConnectionError("no more sockets")
return remaining.pop(0)
kwargs.setdefault("heartbeat_interval_s", None)
kwargs.setdefault("reconnect", None)
kwargs.setdefault("handshake_timeout_s", 5.0)
client = DuplexClient("ws://test-host:8099", model="test-model", connect=connect, **kwargs)
return client, calls
# ---------------------------------------------------------------------------
# SessionConfig
def test_session_config_payload_defaults():
payload = SessionConfig().to_session_payload(model="m", session_id="s")
assert payload["model"] == "m"
assert payload["session_id"] == "s"
assert payload["modalities"] == ["audio", "text"]
assert payload["input_audio_format"] == "pcm16"
assert payload["output_audio_format"] == "pcm16"
# The encodings do not pin the rates; the payload must carry both (the
# server reads the input rate top-level and the output rate from audio).
assert payload["sample_rate_hz"] == 16_000
assert payload["audio"] == {
"input": {"sample_rate_hz": 16_000},
"output": {"sample_rate_hz": 24_000},
}
assert payload["turn_detection"] is None
assert payload["extra_body"] == {"auto_response": True}
assert "voice" not in payload
assert "ref_audio" not in payload
def test_session_config_payload_carries_preset_rates():
config = SessionConfig(
input_audio=AudioFormat("pcm_f32le", 24_000),
output_audio=AudioFormat("pcm16", 24_000),
)
payload = config.to_session_payload(model="m")
assert payload["input_audio_format"] == "pcm_f32le"
assert payload["sample_rate_hz"] == 24_000
assert payload["audio"] == {
"input": {"sample_rate_hz": 24_000},
"output": {"sample_rate_hz": 24_000},
}
def test_audio_format_math():
fmt = AudioFormat("pcm16", 16_000)
assert fmt.byte_count(100) == 3200
assert fmt.duration_ms(3200) == 100.0
f32 = AudioFormat("pcm_f32le", 24_000)
assert f32.byte_count(80) == 1920 * 4
with pytest.raises(ValueError):
_ = AudioFormat("mp3", 16_000).bytes_per_sample
# ---------------------------------------------------------------------------
# Handshake and lifecycle
async def test_handshake_adopts_session_and_acks():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, calls = make_client(sock)
async with client:
assert client.session_id == "sess-1"
assert client.resume_token == "tok-1"
assert calls == ["ws://test-host:8099/v1/realtime?duplex=1&model=test-model&autostart=0"]
assert sock.sent[0]["type"] == "session.update"
assert sock.sent[0]["session"]["model"] == "test-model"
await _drain(lambda: {"type": "session.event_ack", "server_event_seq": 1} in _acks(sock))
close_task = asyncio.create_task(client.close())
await _drain(lambda: "session.close" in sock.sent_types())
sock.feed(SESSION_CLOSED)
await close_task
assert "session.close" in sock.sent_types()
async def test_handshake_error_raises_protocol_error():
sock = FakeSocket()
sock.feed({"type": "error", "error": {"code": "unsupported_audio_format", "message": "bad"}})
client, _ = make_client(sock)
with pytest.raises(DuplexProtocolError) as excinfo:
async with client:
pass
assert excinfo.value.code == "unsupported_audio_format"
async def test_session_expired_raises_from_event_stream():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
sock.feed({"type": "session.expired", "reason": "lease_expired", "server_event_seq": 2})
with pytest.raises(DuplexSessionClosedError) as excinfo:
async for _ in client.events():
pass
assert "lease_expired" in excinfo.value.reason
async def test_resync_required_drops_resume_credential():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, calls = make_client(sock, reconnect=ReconnectPolicy(max_attempts=2, backoff_s=(0.0, 0.0)))
async with client:
assert client.resume_token == "tok-1"
stream = client.events()
sock.feed({"type": "session.resync_required", "session_id": "sess-1", "server_event_seq": 2})
async for event in stream:
if event.type == "session.resync_required":
break
assert client.resume_token is None
# With the credential gone, a transport drop must finalize instead of
# attempting session.resume against a server that stopped journaling.
sock.feed(RuntimeError("transport dropped"))
with pytest.raises(DuplexSessionClosedError):
async for _ in stream:
pass
assert len(calls) == 1
# ---------------------------------------------------------------------------
# Input events
async def test_append_audio_tracks_cumulative_end_ms():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
pcm = b"\x01\x02" * 1600 # 100 ms of pcm16 @ 16 kHz
await client.append_audio(pcm)
await client.append_audio(pcm, is_speech=True)
appends = [event for event in sock.sent if event.get("type") == "input_audio_buffer.append"]
assert [a["audio_end_ms"] for a in appends] == [100, 200]
assert appends[0]["format"] == "pcm16"
assert appends[0]["sample_rate_hz"] == 16_000
assert appends[0]["duration_ms"] == 100
assert "is_speech" not in appends[0]
assert appends[1]["is_speech"] is True
assert base64.b64decode(appends[0]["audio"]) == pcm
sock.feed(SESSION_CLOSED)
async def test_append_audio_strips_video_frame_data_url_prefix():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
jpeg_b64 = base64.b64encode(b"\xff\xd8fake").decode("ascii")
# The wire contract carries bare base64; image_data_url output must
# still be accepted (the server validator rejects data-URL prefixes).
await client.append_audio(b"\x00\x00", video_frames=[f"data:image/jpeg;base64,{jpeg_b64}", jpeg_b64])
append = next(event for event in sock.sent if event.get("type") == "input_audio_buffer.append")
assert append["video_frames"] == [jpeg_b64, jpeg_b64]
sock.feed(SESSION_CLOSED)
async def test_stream_pcm_chunking():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
await client.stream_pcm(b"\x00" * 8000, chunk_ms=100, realtime=False)
appends = [event for event in sock.sent if event.get("type") == "input_audio_buffer.append"]
assert [a["duration_ms"] for a in appends] == [100, 100, 50]
sock.feed(SESSION_CLOSED)
async def test_interruption_primitives_send_documented_events():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
await client.cancel_response()
await client.clear_input()
types = sock.sent_types()
assert types.index("response.cancel") < types.index("input_audio_buffer.clear")
sock.feed(SESSION_CLOSED)
# ---------------------------------------------------------------------------
# Response demultiplexing
def _b64(data: bytes) -> str:
return base64.b64encode(data).decode("ascii")
async def test_response_handle_flow():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
chunk = b"\x00\x01" * 2400 # 100 ms of pcm16 @ 24 kHz
client, _ = make_client(sock)
async with client:
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
sock.feed(
{
"type": "response.audio.delta",
"response_id": "resp-1",
"delta": _b64(chunk),
"sample_rate_hz": 24_000,
"server_event_seq": 4,
}
)
sock.feed(
{
"type": "response.audio_transcript.delta",
"response_id": "resp-1",
"delta": "hi there",
"server_event_seq": 5,
}
)
sock.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 6})
async for response in client.responses():
chunks = [piece async for piece in response.audio()]
assert chunks == [chunk]
assert response.decision == "speak"
assert response.transcript == "hi there"
assert response.played_ms == pytest.approx(100.0)
assert response.finished
break
sock.feed(SESSION_CLOSED)
async def test_listen_decision_yields_finished_silent_handle():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
sock.feed({"type": "response.listen", "server_event_seq": 2})
async for response in client.responses():
assert response.decision == "listen"
assert response.finished
assert [piece async for piece in response.audio()] == []
break
sock.feed(SESSION_CLOSED)
async def test_listen_terminates_active_response():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
sock.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 3})
async for response in client.responses():
await response.wait(timeout_s=5.0)
assert response.decision == "listen"
break
sock.feed(SESSION_CLOSED)
async def test_id_less_listen_never_closes_an_in_flight_response():
# An id-less listen is a standalone decision beat (e.g. a silence-skip
# while a response streams); it must surface as its own finished handle
# and leave the in-flight response to its real terminal.
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
sock.feed({"type": "response.listen", "server_event_seq": 4})
sock.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 5})
seen: list[tuple[str | None, str | None, bool]] = []
async for response in client.responses():
await response.wait(timeout_s=5.0)
seen.append((response.response_id, response.decision, response.finished))
if len(seen) == 2:
break
assert ("resp-1", "speak", True) in seen
assert (None, "listen", True) in seen
sock.feed(SESSION_CLOSED)
async def test_terminal_listen_keeps_spoken_decision():
# A terminal listen after the response spoke is the model yielding the
# turn; the handle must stay decision="speak" with its audio intact.
sock = FakeSocket()
sock.feed(SESSION_CREATED)
chunk = b"\x00\x01" * 2400
client, _ = make_client(sock)
async with client:
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
sock.feed(
{
"type": "response.audio.delta",
"response_id": "resp-1",
"delta": _b64(chunk),
"sample_rate_hz": 24_000,
"server_event_seq": 4,
}
)
sock.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 5})
async for response in client.responses():
chunks = [piece async for piece in response.audio()]
assert chunks == [chunk]
assert response.decision == "speak"
assert response.finished
break
sock.feed(SESSION_CLOSED)
async def test_error_event_surfaces_on_response_path():
# A rejected send produces no response; a consumer waiting in responses()
# must see the rejection instead of waiting forever.
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
sock.feed(
{
"type": "error",
"error": {"code": "input_audio_buffer_empty", "message": "empty commit"},
"server_event_seq": 2,
}
)
with pytest.raises(DuplexProtocolError) as excinfo:
async for _ in client.responses():
pass
assert excinfo.value.code == "input_audio_buffer_empty"
sock.feed(SESSION_CLOSED)
async def test_slow_audio_consumer_drops_oldest_instead_of_stalling():
from vllm_omni.clients.duplex import AudioDelta, ResponseHandle
handle = ResponseHandle(
response_id="r1",
output_format=AudioFormat("pcm16", 24_000),
max_buffered_events=2,
)
for payload in (b"c1", b"c2", b"c3"):
# _feed must never block the reader, even against a full queue.
handle._feed(AudioDelta({"type": "response.audio.delta", "delta": _b64(payload)}))
handle._finish(None)
chunks = [chunk async for chunk in handle.audio()]
assert chunks == [b"c3"] # oldest chunks were dropped, the sentinel landed
# ---------------------------------------------------------------------------
# Resume
async def test_resume_after_transport_drop():
first = FakeSocket()
first.feed(SESSION_CREATED)
second = FakeSocket()
# Mirror the real resume activation payload
# (entrypoints/duplex/serving.py, activation_payload_factory): it carries
# no nested "session" object, only the identity fields.
second.feed(
{
"type": "session.resumed",
"session_id": "sess-1",
"incarnation": 0,
"attachment_generation": 1,
"resume_token": "tok-2",
"server_event_seq": 5,
}
)
client, calls = make_client(
first,
second,
reconnect=ReconnectPolicy(max_attempts=2, backoff_s=(0.0, 0.0)),
)
async with client:
stream = client.events()
async def take_two():
seen = []
async for event in stream:
seen.append(event)
if len(seen) == 2:
return seen
consumer = asyncio.create_task(take_two())
await asyncio.sleep(0) # let the consumer subscribe before the drop
await asyncio.sleep(0)
first.feed(RuntimeError("transport dropped"))
seen = await asyncio.wait_for(consumer, timeout=5.0)
assert isinstance(seen[0], ConnectionResumed)
assert isinstance(seen[1], SessionResumed)
assert client.resume_token == "tok-2"
# The activation payload has no session object; the info captured at
# the original handshake must survive the resume.
assert client.session_info == {"session_id": "sess-1"}
assert len(calls) == 2
resume = second.sent[0]
assert resume["type"] == "session.resume"
assert resume["session_id"] == "sess-1"
assert resume["incarnation"] == 0
assert resume["resume_token"] == "tok-1"
assert resume["last_received_server_event_seq"] == 1
second.feed(SESSION_CLOSED)
async def test_no_reconnect_policy_surfaces_closed():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock) # reconnect=None
async with client:
stream = client.events()
sock.feed(RuntimeError("transport dropped"))
with pytest.raises(DuplexSessionClosedError):
async for _ in stream:
pass
async def test_heartbeat_survives_transient_send_failure():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
armed = {"fail": True}
original_send = sock.send
async def flaky_send(raw: str) -> None:
if json.loads(raw).get("type") == "session.heartbeat" and armed["fail"]:
armed["fail"] = False
raise RuntimeError("transport hiccup")
await original_send(raw)
sock.send = flaky_send # type: ignore[method-assign]
client, _ = make_client(sock, heartbeat_interval_s=0.01)
async with client:
# The first tick fails; the loop must keep ticking (a resumed but
# quiet session relies on heartbeats to reset the server timeout).
await _drain(lambda: "session.heartbeat" in sock.sent_types())
sock.feed(SESSION_CLOSED)
async def test_aexit_sends_session_close_on_error_path():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
with pytest.raises(RuntimeError):
async with client:
raise RuntimeError("application failure")
# The server must not be left holding the session for the disconnect
# grace period; the error path still announces the close.
assert "session.close" in sock.sent_types()
def test_target_url_forces_autostart_off():
client = DuplexClient("ws://test-host:8099/v1/realtime?autostart=1", model="m")
query = parse_qs(urlsplit(client._target_url()).query)
# autostart would race the session.update handshake and silently drop
# ref_audio/extra_body; the client overrides it even when the URL asks.
assert query["autostart"] == ["0"]
async def test_fatal_resume_error_gives_up():
first = FakeSocket()
first.feed(SESSION_CREATED)
second = FakeSocket()
second.feed({"type": "error", "error": {"code": "invalid_resume_token", "message": "nope"}})
client, calls = make_client(
first,
second,
reconnect=ReconnectPolicy(max_attempts=3, backoff_s=(0.0, 0.0)),
)
async with client:
stream = client.events()
first.feed(RuntimeError("transport dropped"))
with pytest.raises(DuplexSessionClosedError):
async for _ in stream:
pass
assert len(calls) == 2 # no retry after a fatal resume error
# ---------------------------------------------------------------------------
# Collector
def test_event_collector_accumulates_audio():
collector = EventCollector()
collector.add({"type": "response.created", "response": {"id": "r1"}}, received_at_s=1.0)
collector.add(
{"type": "response.audio.delta", "response_id": "r1", "delta": _b64(b"ab"), "sample_rate_hz": 16_000},
received_at_s=1.1,
)
collector.add(
{"type": "response.output_audio.delta", "response_id": "r1", "audio": _b64(b"cd")},
received_at_s=1.2,
)
assert collector.count("response.created") == 1
assert collector.audio_bytes() == b"abcd"
assert collector.output_sample_rate_hz == 16_000
summary = collector.timing_summary(after_s=0.0)
assert summary["audio_output"]["chunk_count"] == 2
assert summary["audio_output"]["response_created_to_first_audio_ms"] == pytest.approx(100.0)
# ---------------------------------------------------------------------------
# helpers
def _acks(sock: FakeSocket) -> list[dict[str, object]]:
return [event for event in sock.sent if event.get("type") == "session.event_ack"]
async def _drain(predicate, *, timeout_s: float = 2.0) -> None:
deadline = asyncio.get_event_loop().time() + timeout_s
while not predicate():
if asyncio.get_event_loop().time() > deadline:
raise AssertionError("condition not reached")
await asyncio.sleep(0.01)
# ---------------------------------------------------------------------------
# Probe/benchmark helpers (ported from the retired MiniCPM demo client tests)
def test_build_realtime_url_with_model_extra_query():
url = build_realtime_url(
"ws://localhost:8099/v1/realtime?custom=1",
"openbmb/MiniCPM-o-4_5",
session_id="session-a",
extra_query={"native_duplex": "1"},
)
query = parse_qs(urlsplit(url).query)
assert query == {
"custom": ["1"],
"duplex": ["1"],
"model": ["openbmb/MiniCPM-o-4_5"],
"native_duplex": ["1"],
"session_id": ["session-a"],
}
def test_build_realtime_url_resume_only_when_autostart_disabled():
url = build_realtime_url(
"ws://localhost:8099/v1/realtime?duplex=1",
"openbmb/MiniCPM-o-4_5",
autostart=False,
extra_query={"native_duplex": "1"},
)
query = parse_qs(urlsplit(url).query)
assert query["autostart"] == ["0"]
assert query["native_duplex"] == ["1"]
def test_event_collector_partitions_audio_by_response():
collector = EventCollector()
collector.add({"type": "response.created", "response": {"id": "resp-a"}})
collector.add(
{
"type": "response.audio.delta",
"response_id": "resp-a",
"delta": base64.b64encode(b"audio-a").decode("ascii"),
"sample_rate_hz": 16_000,
}
)
assert collector.response_ids == ["resp-a"]
assert collector.audio_bytes("resp-a") == b"audio-a"
assert collector.output_sample_rate_hz == 16_000
assert collector.first_received_at("response.created") is not None
assert collector.last_received_at("response.audio.delta") is not None
def test_event_collector_reports_engine_token_and_audio_intervals():
collector = EventCollector()
collector.add(
{"type": "response.created", "response": {"id": "resp-a"}},
received_at_s=10.0,
)
stage_metrics = {
"0": {
"num_tokens_out": 4,
"vllm_ttft_ms": 120.0,
"vllm_tpot_ms": 15.0,
"vllm_itl_ms": 14.0,
"vllm_itls_ms": [10.0, 14.0, 18.0],
}
}
for received_at_s, cumulative_audio_ms in ((10.2, 80), (10.25, 160), (10.36, 240)):
collector.add(
{
"type": "response.audio.delta",
"response_id": "resp-a",
"delta": base64.b64encode(b"audio").decode("ascii"),
"sample_rate_hz": 16_000,
"metadata": {
"audio_duration_ms": cumulative_audio_ms,
"vllm_omni": {"stage_metrics": stage_metrics},
},
},
received_at_s=received_at_s,
)
collector.add(
{"type": "response.audio_transcript.delta", "response_id": "resp-a", "delta": ""},
received_at_s=10.1,
)
collector.add(
{"type": "response.audio_transcript.delta", "response_id": "resp-a", "delta": "hello"},
received_at_s=10.15,
)
collector.add(
{"type": "response.done", "response": {"id": "resp-a"}},
received_at_s=10.4,
)
timing = collector.timing_summary(
after_s=10.0,
input_committed_at_s=9.9,
response_id="resp-a",
)
assert timing["stage0_tokens"] == {
"source": "engine_stage_metrics",
"output_token_count": 4,
"ttft_ms": 120.0,
"tpot_ms": 15.0,
"itls_ms": [10.0, 14.0, 18.0],
"inter_token_interval_ms": {
"count": 3,
"mean": 14.0,
"p50": 14.0,
"p95": 18.0,
"max": 18.0,
},
}
assert timing["audio_output"] == {
"source": "client_monotonic_receive",
"chunk_count": 3,
"response_created_to_first_audio_ms": 200.0,
"commit_to_first_audio_ms": 300.0,
"inter_chunk_interval_ms": {
"count": 2,
"mean": 80.0,
"p50": 50.0,
"p95": 110.0,
"max": 110.0,
},
"chunk_duration_ms": {
"count": 3,
"mean": 80.0,
"p50": 80.0,
"p95": 80.0,
"max": 80.0,
},
"max_chunk_gap_ms": 110.0,
}
# Raw data only: derived metrics such as the RTF are the caller's job
# (e.g. vllm_omni.metrics.definitions.compute_audio_rtf).
assert timing["request_metrics"] == {
"source": "client_monotonic_receive",
"measurement_origin": {
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
},
"ttft_ms": 250.0,
"ttfp_ms": 300.0,
"audio_generation_ms": 460.0,
"audio_duration_ms": 240.0,
}
def test_response_timing_ignores_unowned_session_level_metrics():
collector = EventCollector()
collector.add(
{"type": "response.created", "response": {"id": "resp-a"}},
received_at_s=10.0,
)
collector.add(
{
"type": "response.audio.delta",
"response_id": "resp-a",
"delta": base64.b64encode(b"audio").decode("ascii"),
"metadata": {
"vllm_omni": {
"stage_metrics": {
"0": {
"num_tokens_out": 20,
"vllm_ttft_ms": 157.0,
"vllm_tpot_ms": 16.0,
"vllm_itls_ms": [15.0, 17.0],
}
}
}
},
},
received_at_s=10.2,
)
collector.add(
{
"type": "response.listen",
"metadata": {
"vllm_omni": {
"stage_metrics": {
"0": {
"num_tokens_out": 2,
"vllm_ttft_ms": 106.0,
"vllm_tpot_ms": 0.0,
"vllm_itls_ms": [],
}
}
}
},
},
received_at_s=10.3,
)
timing = collector.timing_summary(after_s=10.0, response_id="resp-a")
assert timing["stage0_tokens"]["output_token_count"] == 20
assert timing["stage0_tokens"]["ttft_ms"] == 157.0
def test_summarize_session_request_metrics_averages_audio_turns():
summary = summarize_session_request_metrics(
[
{"ttft_ms": 100.0, "ttfp_ms": 200.0, "rtf": 0.5},
{"ttft_ms": 300.0, "ttfp_ms": 400.0, "rtf": 0.7},
],
session_id="sess-1",
)
assert summary == {
"session_id": "sess-1",
"audio_turn_count": 2,
"mean_ttft_ms": 200.0,
"mean_ttfp_ms": 300.0,
"mean_rtf": 0.6,
}
def test_pcm16_wav_round_trip(tmp_path):
path = tmp_path / "audio.wav"
pcm16 = b"\x01\x00\x02\x00"
write_pcm16_wav(path, pcm16, sample_rate_hz=16_000)
with wave.open(str(path), "rb") as wav_file:
assert wav_file.getnchannels() == 1
assert wav_file.getframerate() == 16_000
assert read_pcm16_wav(path) == pcm16
# ---------------------------------------------------------------------------
# Model-unit helpers and camera-frame interleaving
def test_duplex_unit_boundary_and_residual_math():
assert duplex_unit_boundary_ms(0) == 1030
assert duplex_unit_boundary_ms(2) == 3030
assert has_residual_model_unit(b"\x00" * 32_000, chunk_period_ms=1000) is False
assert has_residual_model_unit(b"\x00" * 32_002, chunk_period_ms=1000) is True
created = {"type": "session.created", "session": {"capabilities": {"chunk_period_ms": 500}}}
assert chunk_period_ms([created]) == 500
assert chunk_period_ms([{"type": "session.created", "session": {}}]) == 1000
async def test_stream_pcm_sends_each_units_composite_beside_its_base_frame():
sock = FakeSocket()
sock.feed(SESSION_CREATED)
client, _ = make_client(sock)
async with client:
frames_sent = await client.stream_pcm(
b"\x01\x00" * (16_000 * 3),
chunk_ms=200,
realtime=False,
video_frames=["f0", "f1"],
stacked_video_frames=["s0", None],
)
appends = [event for event in sock.sent if event.get("type") == "input_audio_buffer.append"]
# A composite belongs to the unit it was captured in, so it rides the
# same append as that unit's base frame; a unit without one sends the
# base alone. Frame k rides the append that closes model unit k.
assert [event["video_frames"] for event in appends if "video_frames" in event] == [["f0", "s0"], ["f1"]]
assert [event["audio_end_ms"] for event in appends if "video_frames" in event] == [1200, 2200]
assert frames_sent == 2
sock.feed(SESSION_CLOSED)
def test_build_realtime_url_native_duplex_flag_and_http_scheme():
url = build_realtime_url("http://localhost:8099/v1/realtime", None, native_duplex=None)
parts = urlsplit(url)
assert parts.scheme == "ws"
assert parse_qs(parts.query) == {"duplex": ["1"]}
url = build_realtime_url("https://host/v1/realtime", "m", native_duplex=False)
parts = urlsplit(url)
assert parts.scheme == "wss"
assert parse_qs(parts.query)["native_duplex"] == ["0"]
with pytest.raises(ValueError):
build_realtime_url("ftp://host/v1/realtime", "m")
def test_reference_audio_data_url(tmp_path):
assert reference_audio_data_url(None) is None
path = tmp_path / "ref.wav"
path.write_bytes(b"RIFF")
assert reference_audio_data_url(str(path)) == "data:audio/wav;base64," + base64.b64encode(b"RIFF").decode("ascii")
with pytest.raises(FileNotFoundError):
reference_audio_data_url(str(tmp_path / "missing.wav"))
def test_event_collector_response_text_joins_deltas_per_response():
collector = EventCollector()
collector.add({"type": "response.created", "response_id": "r1"})
collector.add({"type": "response.audio_transcript.delta", "response_id": "r1", "delta": "he"})
collector.add({"type": "response.output_text.delta", "response_id": "r2", "delta": "other"})
collector.add({"type": "response.text.delta", "response_id": "r1", "delta": "llo"})
assert collector.response_text("r1") == "hello"
assert collector.response_text("r2") == "other"
assert collector.response_text("r3") == ""
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Model duplex session presets under ``vllm_omni.clients.<model>``."""
import subprocess
import sys
import pytest
from vllm_omni.clients.duplex import AudioFormat
from vllm_omni.clients.minicpmo_4_5 import (
create_duplex_session_config as create_minicpmo45_session_config,
)
from vllm_omni.clients.personaplex import (
create_duplex_session_config as create_personaplex_session_config,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_minicpmo45_session_config_matches_native_duplex_deployment():
config = create_minicpmo45_session_config(ref_audio="data:audio/wav;base64,AAA=", temperature=0.0)
payload = config.to_session_payload(model="openbmb/MiniCPM-o-4_5")
assert payload["ref_audio"] == "data:audio/wav;base64,AAA="
assert payload["overlap_policy"] == "listen_only"
assert payload["playback_commit_policy"] == "ack_only"
assert payload["extra_body"]["native_duplex"] is True
assert payload["extra_body"]["auto_response"] is True
assert payload["temperature"] == 0.0
def test_personaplex_session_config_matches_deployment():
config = create_personaplex_session_config(voice="NATF2.pt", persona="You are calm.")
assert config.input_audio == AudioFormat("pcm_f32le", 24_000)
payload = config.to_session_payload(model="nvidia/personaplex-7b-v1")
assert payload["input_audio_format"] == "pcm_f32le"
assert payload["voice"] == "NATF2.pt"
assert payload["instructions"] == "You are calm."
@pytest.mark.parametrize(
"duplex_package",
[
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
"vllm_omni.model_executor.models.personaplex.duplex",
],
)
def test_model_duplex_packages_stay_clear_of_client_library(duplex_package: str):
"""Model duplex plugin packages are server-side: no vllm_omni.clients."""
script = f"""
import sys
import {duplex_package}
loaded = [
name
for name in sys.modules
if name == "vllm_omni.clients" or name.startswith("vllm_omni.clients.")
]
if loaded:
raise SystemExit("model duplex package loaded the client library: " + ", ".join(loaded))
"""
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
timeout=120,
check=False,
)
assert result.returncode == 0, result.stderr
@@ -1,30 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import subprocess
import sys
def test_batched_runtime_import_does_not_require_optional_opus_dependency() -> None:
script = """
import builtins
real_import = builtins.__import__
def guarded_import(name, *args, **kwargs):
if name == "sphn":
raise ModuleNotFoundError("blocked optional dependency: sphn")
return real_import(name, *args, **kwargs)
builtins.__import__ = guarded_import
from vllm_omni.experimental.fullduplex.personaplex.serving.batched import BatchedSessionManager
assert BatchedSessionManager is not None
"""
result = subprocess.run(
[sys.executable, "-c", script],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
+7 -7
View File
@@ -15,20 +15,20 @@ from vllm_omni.experimental.fullduplex.client import (
read_pcm16_wav,
write_pcm16_wav,
)
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
MiniCPMO45DuplexPolicy,
)
from vllm_omni.experimental.fullduplex.video_stacking import (
concat_frames,
unit_subframe_offsets,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
MiniCPMO45DuplexPolicy,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_realtime_client_builds_explicit_native_duplex_url():
url = build_realtime_url(
"ws://localhost:8099/v1/realtime?custom=1&duplex=0&model=stale&minicpmo45_native_duplex=0&session_id=stale",
"ws://localhost:8099/v1/realtime?custom=1&duplex=0&model=stale&native_duplex=0&session_id=stale",
"openbmb/MiniCPM-o-4_5",
session_id="session-a",
)
@@ -38,7 +38,7 @@ def test_realtime_client_builds_explicit_native_duplex_url():
"custom": ["1"],
"duplex": ["1"],
"model": ["openbmb/MiniCPM-o-4_5"],
"minicpmo45_native_duplex": ["1"],
"native_duplex": ["1"],
"session_id": ["session-a"],
}
@@ -65,7 +65,7 @@ def test_realtime_client_builds_resume_only_url_when_autostart_disabled():
query = parse_qs(urlsplit(url).query)
assert query["autostart"] == ["0"]
assert query["minicpmo45_native_duplex"] == ["1"]
assert query["native_duplex"] == ["1"]
@pytest.mark.asyncio
@@ -232,7 +232,7 @@ async def test_realtime_client_configure_explicit_tts_opts_out_of_native_duplex(
session_extra_body = client.sent[0]["session"]["extra_body"]
assert session_extra_body == {
"ref_audio": "data:audio/wav;base64,AAAA",
"minicpmo45_native_duplex": False,
"native_duplex": False,
}
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Contract tests for the fullduplex core (PR #3907 alignment).
Covers the DuplexFence identity value, the session identity rules (epoch only
@@ -14,6 +14,7 @@ import dataclasses
import pytest
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.experimental.fullduplex.core import protocol as ev
from vllm_omni.experimental.fullduplex.core.adapter import (
DuplexAdapter,
@@ -26,7 +27,6 @@ from vllm_omni.experimental.fullduplex.core.session import (
DuplexSessionConfig,
DuplexState,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,262 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PersonaPlex adapter driven through the model-owned lockstep runtime.
PersonaPlexDuplexRuntime starts ONE response for the whole session, streams one
agent frame per user frame, and drains on close (core stays verbatim upstream;
the lockstep lifecycle is model policy) — all with a GPU-free stub stepper.
"""
import asyncio
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.core import protocol as ev
from vllm_omni.experimental.fullduplex.core.session import (
DuplexSession,
DuplexSessionConfig,
DuplexState,
)
from vllm_omni.experimental.fullduplex.personaplex.adapter import (
PersonaPlexDuplexAdapter,
PersonaPlexDuplexRuntime,
)
from vllm_omni.experimental.fullduplex.personaplex.config import FRAME_SIZE
from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class StubStepper:
"""Deterministic FrameStepper: frame i -> audio filled with i, text 't{i}' on odds."""
sample_rate = 24000
frame_size = FRAME_SIZE
def __init__(self) -> None:
self.opened: tuple[str | None, str | None] | None = None
self.steps = 0
def open_session(self, voice_prompt=None, persona=None) -> None:
self.opened = (voice_prompt, persona)
self.steps = 0
def step(self, user_pcm):
assert user_pcm.shape[0] == self.frame_size
self.steps += 1
i = self.steps
return FrameOutput(
audio=np.full(self.frame_size, float(i), dtype=np.float32),
text=(f"t{i}" if i % 2 else None),
)
class RecordingAdapter(PersonaPlexDuplexAdapter):
def __init__(self, stepper: StubStepper) -> None:
super().__init__(stepper)
self.closed = False
async def on_close(self, session: DuplexSession) -> None:
self.closed = True
await super().on_close(session)
def _collector():
out: list[dict] = []
async def emit(event: dict) -> None:
out.append(event)
return out, emit
async def _feed(events):
for e in events:
yield e
async def _raising_feed(frame):
yield {"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}
await asyncio.sleep(0)
raise RuntimeError("input failed")
def _respond_tasks() -> list[asyncio.Task]:
current = asyncio.current_task()
return [
task
for task in asyncio.all_tasks()
if task is not current and "_respond_once" in getattr(task.get_coro(), "__qualname__", "")
]
def _session():
cfg = DuplexSessionConfig(
input_modalities=("audio",),
output_modalities=("audio", "text"),
)
return DuplexSession("s", cfg)
@pytest.mark.asyncio
async def test_lockstep_one_agent_frame_per_user_frame():
stub = StubStepper()
adapter = PersonaPlexDuplexAdapter(stub, voice_prompt="NATF2.pt", persona="be terse")
rt = PersonaPlexDuplexRuntime(_session(), adapter)
out, emit = _collector()
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame} for _ in range(3)]
events.append({"type": ev.CLOSE})
await rt.run(_feed(events), emit)
types = [e["type"] for e in out]
# exactly one response for the whole session, drained cleanly on close
assert types.count(ev.RESPONSE_CREATED) == 1
assert types.count(ev.RESPONSE_DONE) == 1
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
text = [e["data"] for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "text"]
assert len(audio) == 3 # one agent frame per user frame
assert [int(e["data"][0]) for e in audio] == [1, 2, 3] # lockstep order
assert text == ["t1", "t3"]
assert stub.steps == 3
assert stub.opened == ("NATF2.pt", "be terse")
@pytest.mark.asyncio
async def test_unaligned_chunks_are_reframed_to_80ms():
stub = StubStepper()
adapter = PersonaPlexDuplexAdapter(stub)
rt = PersonaPlexDuplexRuntime(_session(), adapter)
out, emit = _collector()
# 2.5 frames in one chunk + 0.5 frame in another => 3 whole frames
events = [
{"type": ev.INPUT_APPEND, "modality": "audio", "data": np.zeros(FRAME_SIZE * 5 // 2, dtype=np.float32)},
{"type": ev.INPUT_APPEND, "modality": "audio", "data": np.zeros(FRAME_SIZE // 2, dtype=np.float32)},
{"type": ev.CLOSE},
]
await rt.run(_feed(events), emit)
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
assert len(audio) == 3
assert stub.steps == 3
@pytest.mark.asyncio
async def test_close_without_input_is_clean():
adapter = PersonaPlexDuplexAdapter(StubStepper())
rt = PersonaPlexDuplexRuntime(_session(), adapter)
out, emit = _collector()
await rt.run(_feed([{"type": ev.CLOSE}]), emit)
# no input -> response never started -> no created/done, no error
assert not any(e["type"] == ev.ERROR for e in out)
assert not any(e["type"] == ev.RESPONSE_CREATED for e in out)
@pytest.mark.asyncio
async def test_iterator_eof_without_close_drains_and_stops():
# Transport drop: the input iterator ends WITHOUT an explicit CLOSE event.
# The runtime must still signal on_close so the eternal response drains
# instead of blocking on its inbox forever.
stub = StubStepper()
adapter = PersonaPlexDuplexAdapter(stub)
rt = PersonaPlexDuplexRuntime(_session(), adapter)
out, emit = _collector()
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}]
await asyncio.wait_for(rt.run(_feed(events), emit), timeout=5.0)
types = [e["type"] for e in out]
assert types.count(ev.RESPONSE_DONE) == 1
assert stub.steps == 1
@pytest.mark.asyncio
async def test_input_exception_closes_response_and_runtime() -> None:
stub = StubStepper()
adapter = RecordingAdapter(stub)
rt = PersonaPlexDuplexRuntime(_session(), adapter)
out, emit = _collector()
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
with pytest.raises(RuntimeError, match="input failed"):
await rt.run(_raising_feed(frame), emit)
leaked = _respond_tasks()
for task in leaked:
task.cancel()
if leaked:
await asyncio.gather(*leaked, return_exceptions=True)
assert adapter.closed is True
assert rt.session.state is DuplexState.CLOSED
assert not leaked
@pytest.mark.asyncio
async def test_default_session_config_gets_lockstep_lifecycle():
# The lifecycle comes from the model-owned runtime class, not a session
# flag: a plain default-constructed session config runs lockstep.
stub = StubStepper()
adapter = PersonaPlexDuplexAdapter(stub)
session = DuplexSession(
"s",
DuplexSessionConfig(input_modalities=("audio",), output_modalities=("audio", "text")),
)
rt = PersonaPlexDuplexRuntime(session, adapter)
out, emit = _collector()
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}, {"type": ev.CLOSE}]
await rt.run(_feed(events), emit)
types = [e["type"] for e in out]
assert types.count(ev.RESPONSE_CREATED) == 1
assert types.count(ev.RESPONSE_DONE) == 1
assert stub.steps == 1
@pytest.mark.asyncio
async def test_partial_tail_is_padded_and_stepped_on_close():
# A trailing sub-frame chunk is zero-padded and stepped at close (same
# contract as PersonaPlexSession.flush) rather than silently dropped.
stub = StubStepper()
adapter = PersonaPlexDuplexAdapter(stub)
rt = PersonaPlexDuplexRuntime(_session(), adapter)
out, emit = _collector()
events = [
{"type": ev.INPUT_APPEND, "modality": "audio", "data": np.ones(FRAME_SIZE // 2, dtype=np.float32)},
{"type": ev.CLOSE},
]
await rt.run(_feed(events), emit)
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
assert len(audio) == 1
assert stub.steps == 1
@pytest.mark.asyncio
async def test_single_eternal_response_across_barge_in():
# Moved from the core contract tests when the lockstep lifecycle became
# model-owned: ONE response for the whole session, RESPONSE_CANCEL bumps the
# epoch fence (playback cursor) but the stream keeps flowing, and close
# drains via on_close.
stub = StubStepper()
adapter = PersonaPlexDuplexAdapter(stub)
session = _session()
rt = PersonaPlexDuplexRuntime(session, adapter)
out, emit = _collector()
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
events = [
{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame},
{"type": ev.RESPONSE_CANCEL}, # barge-in: epoch bumps, stream continues
{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame},
{"type": ev.CLOSE},
]
await rt.run(_feed(events), emit)
types = [e["type"] for e in out]
assert types.count(ev.RESPONSE_CREATED) == 1, "exactly one eternal response"
assert types.count(ev.RESPONSE_DONE) == 1
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
assert len(audio) == 2, "stream keeps flowing across barge-in"
assert session.epoch == 1, "cancel still advanced the epoch fence"
assert stub.steps == 2
@@ -1,178 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU-free tests for the batched PersonaPlex slot manager (stub engine)."""
from types import SimpleNamespace
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput, PrefillStep
from vllm_omni.experimental.fullduplex.personaplex.serving.batched import (
BatchedSessionManager,
SlotPhase,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
FRAME = 4
class StubEngine:
"""Records the slot-API call sequence; echoes each row's pcm as its output."""
frame_size = FRAME
def __init__(self, batch_size: int = 4, prefill_len: int = 2) -> None:
self.config = SimpleNamespace(batch_size=batch_size)
self.prefill_len = prefill_len
self.recycled: list[int] = []
self.voice_ended: list[int] = []
self.prefill_done: list[int] = []
self.steps: list[tuple[np.ndarray, dict | None]] = []
def prefill_steps(self, persona=None):
steps = [PrefillStep("sacrifice"), PrefillStep("voice", embedding=None), PrefillStep("voice_end")]
steps += [PrefillStep("tokens", moshi_tokens=None, text_token=3, user_sine=True)] * self.prefill_len
return steps
def recycle_slot(self, b):
self.recycled.append(b)
def voice_end(self, b):
self.voice_ended.append(b)
def end_prefill(self, b):
self.prefill_done.append(b)
def step_batch(self, pcm, prefill=None):
self.steps.append((pcm.copy(), dict(prefill) if prefill else None))
return [FrameOutput(audio=pcm[b].copy(), text=None) for b in range(self.config.batch_size)]
def test_all_idle_does_not_step():
eng = StubEngine()
mgr = BatchedSessionManager(eng)
assert mgr.tick() is False
assert eng.steps == []
def test_pristine_slot_is_live_immediately():
eng = StubEngine()
mgr = BatchedSessionManager(eng)
slot, ready_now = mgr.acquire()
assert ready_now is True
assert slot.phase is SlotPhase.LIVE
assert eng.recycled == []
def test_custom_persona_prefills_and_recycles_on_tick_thread():
eng = StubEngine(prefill_len=2)
mgr = BatchedSessionManager(eng)
slot, ready_now = mgr.acquire(persona="custom")
assert ready_now is False
assert slot.phase is SlotPhase.PREFILL
assert eng.recycled == [] # deferred: reset must run between engine steps
ready = []
slot.on_ready = lambda: ready.append(True)
mgr.tick() # recycle + sacrifice
assert eng.recycled == [slot.index]
assert eng.steps[-1][1][slot.index].kind == "sacrifice"
mgr.tick() # voice frame
assert eng.steps[-1][1][slot.index].kind == "voice"
assert eng.voice_ended == []
mgr.tick() # voice_end executed BETWEEN ticks, then first tokens frame
assert eng.voice_ended == [slot.index]
assert eng.steps[-1][1][slot.index].kind == "tokens"
mgr.tick() # second tokens frame
assert not ready
mgr.tick() # prefill exhausted -> live
assert eng.prefill_done == [slot.index]
assert slot.phase is SlotPhase.LIVE
assert ready == [True]
def test_pristine_slot_after_ticking_requires_recycle():
# step_batch advances EVERY row each tick, so a pristine idle row that has
# lived through ticks has drifted from its boot prefill and must be
# recycled before a client may join it.
eng = StubEngine()
mgr = BatchedSessionManager(eng)
first, ready_now = mgr.acquire()
assert ready_now is True # claimed before any tick: truly pristine
mgr.tick()
second, ready_now2 = mgr.acquire()
assert second.index != first.index
assert ready_now2 is False
assert second.phase is SlotPhase.PREFILL
def test_used_slot_requires_recycle_even_with_default_persona():
eng = StubEngine()
mgr = BatchedSessionManager(eng)
slot, ready_now = mgr.acquire()
assert ready_now
mgr.release(slot)
slot2, ready_now2 = mgr.acquire()
assert slot2.index == slot.index
assert ready_now2 is False
assert slot2.phase is SlotPhase.PREFILL
def test_acquire_returns_none_when_full():
eng = StubEngine(batch_size=2)
mgr = BatchedSessionManager(eng)
assert mgr.acquire() is not None
assert mgr.acquire() is not None
assert mgr.acquire() is None
def test_feed_frames_epoch_guard_and_backlog_cap():
eng = StubEngine()
mgr = BatchedSessionManager(eng, max_buffered_frames=3)
slot, _ = mgr.acquire()
mgr.feed(slot, np.arange(FRAME * 2, dtype=np.float32), slot.epoch)
assert len(slot.inq) == 2
mgr.feed(slot, np.zeros(3, dtype=np.float32), slot.epoch) # partial tail buffered
assert len(slot.inq) == 2 and slot.pending.shape[0] == 3
mgr.feed(slot, np.zeros(FRAME * 5, dtype=np.float32), slot.epoch)
assert len(slot.inq) == 3 # oldest dropped beyond the cap
mgr.feed(slot, np.zeros(FRAME, dtype=np.float32), slot.epoch - 1) # stale epoch ignored
assert len(slot.inq) == 3
def test_live_slot_consumes_input_and_emits_output():
eng = StubEngine()
mgr = BatchedSessionManager(eng)
slot, _ = mgr.acquire()
got = []
slot.on_output = got.append
frame = np.full(FRAME, 0.5, dtype=np.float32)
mgr.feed(slot, frame, slot.epoch)
mgr.tick()
assert len(got) == 1
np.testing.assert_array_equal(got[0].audio, frame)
# queue drained: next tick feeds zeros
mgr.tick()
np.testing.assert_array_equal(got[1].audio, np.zeros(FRAME, dtype=np.float32))
def test_release_mid_flight_stops_emission():
eng = StubEngine()
mgr = BatchedSessionManager(eng)
slot, _ = mgr.acquire()
got = []
slot.on_output = got.append
mgr.tick()
assert len(got) == 1
mgr.release(slot)
# another slot keeps the batch non-idle
other, _ = mgr.acquire() # same slot index, now PREFILL (used)
assert other.phase is SlotPhase.PREFILL
mgr.tick()
assert len(got) == 1 # no further output for the released epoch
@@ -1,245 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""GPU e2e for PersonaPlex elastic batching (bit-exact isolation + recycle).
Greedy decoding makes every cross-slot leak observable as a bit difference:
1. Isolation (bit-exact): recycling one slot mid-flight leaves every other
slot's output bit-identical to a run that never recycled.
2. Replay exactness: a recycled slot's opening live ticks match a pristine boot
at numerical-noise level (a broken KV mask / staging roll garbles frame 0).
3. Coherence (behavioral): a recycled slot fed a real spoken question produces
a real reply in its inner-monologue text stream.
Run (moshi-free native stack; ~30 GB GPU + HF access to the gated repo; set
PPLEX_QUESTION_WAV to a spoken-question wav to enable the coherence test):
pytest tests/e2e/features/fullduplex/test_personaplex_elastic.py -v -s
Bench (B sweep, ms/tick vs the 80 ms budget):
python tests/e2e/features/fullduplex/test_personaplex_elastic.py --bench 2 4 8 16
"""
from __future__ import annotations
import numpy as np
import pytest
from tests.helpers.mark import hardware_marks
torch = pytest.importorskip("torch")
# slow + H100 + omni + cards_1 routes this module into weekly
# "Omni · H100 · Single-GPU" (test-weekly.yml).
pytestmark = [
pytest.mark.omni,
pytest.mark.slow,
*hardware_marks(res={"cuda": "H100"}),
]
if not torch.cuda.is_available(): # noqa: E402
pytest.skip("needs CUDA", allow_module_level=True)
from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig # noqa: E402
from vllm_omni.experimental.fullduplex.personaplex.runtime import ( # noqa: E402
PersonaPlexEngine,
)
B = 4
LIVE_TICKS = 40 # live frames compared per claim
RECYCLE_AT = 10
def _make_engine() -> PersonaPlexEngine:
cfg = PersonaPlexConfig(batch_size=B) # native stepper is greedy-only
try:
return PersonaPlexEngine(cfg).load()
except Exception as exc: # gated-repo access is environment-dependent
name = type(exc).__name__
if "Gated" in name or "401" in str(exc) or "403" in str(exc):
pytest.skip(f"no access to the gated nvidia/personaplex-7b-v1 repo: {name}")
raise
@pytest.fixture(scope="module")
def engine() -> PersonaPlexEngine:
return _make_engine()
def _slot_stream(slot: int, tick: int, frame: int) -> np.ndarray:
"""Deterministic per-slot input: distinct low-amplitude tones."""
sr = 24000
freq = 180.0 + 90.0 * slot
t0 = tick * frame
t = (np.arange(frame) + t0) / sr
return (0.12 * np.sin(2 * np.pi * freq * t)).astype(np.float32)
def _run(engine, total_ticks: int, recycle: bool):
"""Run the lockstep loop; optionally recycle slot 1 at RECYCLE_AT.
Returns per-slot lists of (audio | None) per live tick index, where the
recycled slot's list restarts its input stream at its post-prefill tick.
"""
frame = engine.frame_size
outs: list[list[np.ndarray | None]] = [[] for _ in range(B)]
stream_tick = [0] * B # per-slot conversation clock driving its input
prefill = None
tick = 0
while tick < total_ticks:
if recycle and tick == RECYCLE_AT and prefill is None:
engine.recycle_slot(1)
prefill = list(engine.prefill_steps(None))
pcm = np.zeros((B, frame), dtype=np.float32)
step_for_1 = None
if prefill:
while prefill and prefill[0].kind == "voice_end":
engine.voice_end(1)
prefill.pop(0)
if prefill:
step_for_1 = prefill.pop(0)
else:
engine.end_prefill(1)
prefill = None
stream_tick[1] = 0 # restart slot 1's input stream from its own t=0
for b in range(B):
if b == 1 and step_for_1 is not None:
continue # engine overrides the user channel for prefill rows
pcm[b] = _slot_stream(b, stream_tick[b], frame)
stream_tick[b] += 1
frame_outs = engine.step_batch(pcm, {1: step_for_1} if step_for_1 is not None else None)
for b in range(B):
if b == 1 and step_for_1 is not None:
continue # prefill tick: nothing conversational to record
frame_out = frame_outs[b]
outs[b].append(None if frame_out is None else frame_out.audio)
tick += 1
return outs
def test_recycle_isolation_and_fresh_equivalence(engine):
engine.open_batch()
prefill_len = sum(1 for s in engine.prefill_steps(None) if s.kind != "voice_end")
total = RECYCLE_AT + prefill_len + LIVE_TICKS
base = _run(engine, total, recycle=False)
engine.open_batch() # deterministic re-boot (greedy, fixed voice/persona)
rec = _run(engine, total, recycle=True)
# Claim 1 -- isolation: untouched slots bit-identical to the no-recycle run.
for b in (0, 2, 3):
assert len(base[b]) == total and len(rec[b]) == total
for i, (x, y) in enumerate(zip(base[b], rec[b])):
assert (x is None) == (y is None), f"slot {b} tick {i} validity diverged"
if x is not None:
assert np.array_equal(x, y), f"slot {b} tick {i} audio diverged (recycle leaked)"
# Claim 2 -- replay exactness: the recycled slot's opening live ticks match a
# pristine boot at numerical-noise level. A broken KV mask / staging-ring
# roll / Mimi reset garbles the audio from the very first frame (diffs of
# 0.1-1.0); a correct replay differs only by decoder-state rounding (~1e-4).
#
# Full-length bitwise identity is NOT expected, for two structural reasons:
# the replayed system prompt sits at a different ABSOLUTE RoPE offset (bf16
# rounds differently there), and the recycled Mimi ENCODER cannot restore a
# fresh stream's start padding (a batch-shared scalar), so from the first
# non-silent user frame it emits different-but-equally-valid codes for the
# same audio. Once any near-tie flips a token, the two equally valid
# trajectories legitimately diverge; coherence of the recycled slot is
# asserted behaviorally in test_recycled_slot_converses.
fresh = [a for a in base[1] if a is not None][:LIVE_TICKS]
reborn = [a for a in rec[1][RECYCLE_AT:] if a is not None][:LIVE_TICKS]
assert len(reborn) >= LIVE_TICKS // 2, "recycled slot produced too few live frames"
diffs = [float(np.max(np.abs(x - y))) for x, y in zip(fresh, reborn)]
first_flip = next((i for i, d in enumerate(diffs) if d > 5e-2), len(diffs))
print(
f"[elastic] replay exactness: {first_flip}/{len(diffs)} opening ticks at noise level "
f"(head diffs: {[f'{d:.1e}' for d in diffs[:6]]})"
)
assert first_flip >= 4, (
f"recycled slot diverged from fresh behavior at tick {first_flip} "
f"(diffs {diffs[: first_flip + 2]}); the system-prompt replay is broken"
)
assert max(diffs[:4]) < 2e-2, f"reborn audio not at noise level: {max(diffs[:4])}"
def test_recycled_slot_converses(engine):
"""Behavioral guarantee: a recycled slot holds a coherent conversation.
Feed a real spoken question into a slot that was recycled mid-flight (while
other slots keep running); the model's own inner-monologue text stream must
produce a real reply. This is the end-user property the encoder-state
caveats above cannot break.
"""
import os
import sphn
wav_path = os.environ.get("PPLEX_QUESTION_WAV")
if not wav_path:
pytest.skip("set PPLEX_QUESTION_WAV to a spoken-question wav to run")
wav, _ = sphn.read(wav_path, sample_rate=24000)
wav = wav[0].astype(np.float32)
frame = engine.frame_size
pad = (-len(wav)) % frame
wav = np.concatenate([wav, np.zeros(pad, dtype=np.float32)])
q_frames = [wav[i : i + frame] for i in range(0, len(wav), frame)]
engine.open_batch()
pcm = np.zeros((B, frame), dtype=np.float32)
for _ in range(5): # everyone live briefly
engine.step_batch(pcm)
engine.recycle_slot(1)
prefill = list(engine.prefill_steps(None))
while prefill:
while prefill and prefill[0].kind == "voice_end":
engine.voice_end(1)
prefill.pop(0)
if not prefill:
break
engine.step_batch(pcm, {1: prefill.pop(0)})
engine.end_prefill(1)
text = ""
total = len(q_frames) + 120 # question + ~10 s to answer
for i in range(total):
pcm = np.zeros((B, frame), dtype=np.float32)
if i < len(q_frames):
pcm[1] = q_frames[i]
outs = engine.step_batch(pcm)
if outs[1] is not None and outs[1].text:
text += outs[1].text
alpha = sum(c.isalpha() for c in text)
print(f"[elastic] recycled slot inner monologue: {text!r}")
assert alpha >= 15, f"recycled slot produced no coherent reply: {text!r}"
def _bench(batch_sizes: list[int]) -> None:
import time
for bs in batch_sizes:
cfg = PersonaPlexConfig(batch_size=max(bs, 2))
eng = PersonaPlexEngine(cfg).load()
eng.open_batch()
frame = eng.frame_size
pcm = np.zeros((cfg.batch_size, frame), dtype=np.float32)
for _ in range(10): # warm
eng.step_batch(pcm)
torch.cuda.synchronize()
t0 = time.monotonic()
n = 100
for _ in range(n):
eng.step_batch(pcm)
torch.cuda.synchronize()
ms = (time.monotonic() - t0) / n * 1000
mem = torch.cuda.max_memory_allocated() / 2**30
print(f"B={cfg.batch_size:3d} {ms:6.1f} ms/tick RTF/stream {ms / 80:.3f} {mem:.1f} GiB")
del eng
torch.cuda.empty_cache()
if __name__ == "__main__":
import sys
sizes = [int(x) for x in sys.argv[2:]] if len(sys.argv) > 2 and sys.argv[1] == "--bench" else [2, 4, 8]
_bench(sizes)
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Single-session lease teardown: a cancelled engine call must be drained.
Cancelling an asyncio task blocked on a worker-thread engine call returns
immediately while the thread keeps running (executor futures cannot be
interrupted). The server therefore tracks the in-flight future on the session
state and waits it out before releasing the capacity-one lease; otherwise a
fast reconnect could call ``open()`` on the shared engine while the previous
connection's final ``step()`` is still mutating it.
"""
import asyncio
import contextlib
import sys
import threading
import types
import pytest
try:
import sphn # noqa: F401
except ImportError:
# These lease tests never encode/decode Opus. Keep the optional serving
# dependency from hiding dependency-free teardown regressions.
sys.modules["sphn"] = types.ModuleType("sphn")
from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig
from vllm_omni.experimental.fullduplex.personaplex.serving.server import DuplexServer
from vllm_omni.experimental.fullduplex.personaplex.session import PersonaPlexServingSessionState
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.asyncio
async def test_cancelled_engine_call_is_drained_before_lease_release():
server = DuplexServer(PersonaPlexConfig()) # engine never loaded: stub calls only
state = PersonaPlexServingSessionState()
started = threading.Event()
release = threading.Event()
finished = threading.Event()
def slow_engine_op():
started.set()
release.wait(timeout=5.0)
finished.set()
return []
task = asyncio.create_task(server._engine_call(state, slow_engine_op))
await asyncio.to_thread(started.wait, 5.0)
# Cancel while the worker thread is mid-call: the await returns at once,
# the thread keeps running, and its completion event stays tracked on the
# state (the asyncio future itself reads as cancelled/done here, which is
# exactly why the drain must not rely on it).
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
assert state.inflight is not None
assert not state.inflight.is_set()
# The drain must genuinely wait for the worker, not just return.
drain = asyncio.create_task(server._drain_inflight(state))
await asyncio.sleep(0.05)
assert not drain.done()
release.set()
await asyncio.wait_for(drain, timeout=5.0)
assert finished.is_set()
assert state.inflight.is_set()
@pytest.mark.asyncio
async def test_completed_engine_call_clears_inflight():
server = DuplexServer(PersonaPlexConfig())
state = PersonaPlexServingSessionState()
result = await server._engine_call(state, lambda: "ok")
assert result == "ok"
assert state.inflight is None
await server._drain_inflight(state) # no-op on a clean state
@pytest.mark.asyncio
async def test_drain_timeout_keeps_lease_until_worker_finishes():
server = DuplexServer(PersonaPlexConfig())
server._active = True
state = PersonaPlexServingSessionState()
state.inflight = threading.Event()
drained = await server._release_lease_after_drain(state, timeout=0.01)
assert drained is False
assert server._active is True
state.inflight.set()
for _ in range(100):
if not server._active:
break
await asyncio.sleep(0.01)
assert server._active is False
@@ -1,97 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Lockstep-contract tests for the PersonaPlex session driver (no GPU / no moshi).
A stub :class:`FrameStepper` stands in for the real Moshi backend so we can assert
the framing + 1-frame-in / 1-frame-out invariants without weights.
"""
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.personaplex.config import FRAME_SIZE, PersonaPlexConfig
from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput, FrameStepper
from vllm_omni.experimental.fullduplex.personaplex.session import PersonaPlexSession
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class StubStepper:
"""Deterministic FrameStepper: frame i -> audio filled with i, text 't{i}' on odds."""
sample_rate = 24000
frame_size = FRAME_SIZE
def __init__(self) -> None:
self.opened: tuple[str | None, str | None] | None = None
self.steps = 0
def open_session(self, voice_prompt=None, persona=None) -> None:
self.opened = (voice_prompt, persona)
self.steps = 0
def step(self, user_pcm):
assert user_pcm.shape[0] == self.frame_size
self.steps += 1
i = self.steps
return FrameOutput(
audio=np.full(self.frame_size, float(i), dtype=np.float32),
text=(f"t{i}" if i % 2 else None),
)
def test_stub_satisfies_protocol():
assert isinstance(StubStepper(), FrameStepper)
def test_feed_emits_one_output_per_complete_frame():
s = PersonaPlexSession(StubStepper(), PersonaPlexConfig())
s.open(voice_prompt="NATF2.pt", persona="be terse")
out = s.feed(np.zeros(FRAME_SIZE * 3, dtype=np.float32))
assert len(out) == 3
# lockstep order preserved: frame i -> audio filled with i
assert [int(o.audio[0]) for o in out] == [1, 2, 3]
assert [o.text for o in out] == ["t1", None, "t3"]
def test_partial_frames_are_buffered_until_complete():
stub = StubStepper()
s = PersonaPlexSession(stub, PersonaPlexConfig())
s.open()
assert s.feed(np.zeros(FRAME_SIZE // 2, dtype=np.float32)) == [] # nothing yet
assert stub.steps == 0
out = s.feed(np.zeros(FRAME_SIZE, dtype=np.float32)) # 1.5 frames total -> 1 frame
assert len(out) == 1
assert stub.steps == 1
def test_flush_pads_the_tail():
stub = StubStepper()
s = PersonaPlexSession(stub, PersonaPlexConfig())
s.open()
s.feed(np.ones(FRAME_SIZE + 10, dtype=np.float32)) # 1 frame + 10-sample tail
assert stub.steps == 1
tail = s.flush()
assert len(tail) == 1 and stub.steps == 2
assert s.flush() == [] # nothing left
def test_run_concatenates_audio_and_joins_text():
s = PersonaPlexSession(StubStepper(), PersonaPlexConfig())
s.open()
audio, text = s.run(np.zeros(FRAME_SIZE * 4, dtype=np.float32))
assert audio.shape[0] == FRAME_SIZE * 4
assert text == "t1t3" # frames 1..4, text on odds
def test_open_forwards_voice_and_persona():
stub = StubStepper()
s = PersonaPlexSession(stub, PersonaPlexConfig())
s.open(voice_prompt="NATM1.pt", persona="pirate")
assert stub.opened == ("NATM1.pt", "pirate")
def test_feed_before_open_raises():
s = PersonaPlexSession(StubStepper(), PersonaPlexConfig())
with pytest.raises(RuntimeError):
s.feed(np.zeros(FRAME_SIZE, dtype=np.float32))
@@ -15,11 +15,26 @@ import pytest
from huggingface_hub import snapshot_download
from tests.helpers.runtime import OmniServerParams, get_model_prefix
from tests.helpers.stage_config import get_deploy_config_path, get_deploy_duplex_max_sessions
from tests.helpers.stage_config import (
get_deploy_config_path,
get_deploy_duplex_max_sessions,
modify_stage_config,
)
MODEL = "openbmb/MiniCPM-o-4_5"
DEPLOY_CONFIG_REL = "minicpmo_4_5.yaml"
DEPLOY_CONFIG = get_deploy_config_path(DEPLOY_CONFIG_REL)
# Eager-execution variant for fast-startup core-tier probes (e.g. the duplex
# client live test): skips CUDA-graph capture on the LLM and Talker stages.
CORE_DEPLOY_CONFIG = modify_stage_config(
DEPLOY_CONFIG,
updates={
"stages": {
0: {"enforce_eager": True},
1: {"enforce_eager": True},
}
},
)
ASSET_DIR = Path(__file__).resolve().parents[3] / "assets" / "minicpmo_4_5"
RESPONSE_REQUIRED_WAV = ASSET_DIR / "response_required_16k.wav"
RESPONSE_REQUIRED_SHA256 = "2e5fd4eb3ee434ce107ee3a0591fa624a33f7683c7462f45fe651c443c9af941"
@@ -39,6 +54,18 @@ SERVER_PARAMS = [
)
]
CORE_SERVER_PARAMS = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=CORE_DEPLOY_CONFIG,
use_stage_cli=False,
server_args=["--trust-remote-code"],
),
id="three-stage-single-gpu",
)
]
def deploy_max_sessions() -> int:
"""Concurrent duplex sessions the deploy config under test admits.
@@ -52,28 +52,49 @@ try:
except ImportError as exc: # pragma: no cover - demo dependency.
raise SystemExit("Install websockets first: pip install websockets") from exc
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
from vllm_omni.clients.duplex import ( # noqa: E402
PCM16_BYTES_PER_SAMPLE,
PCM16_SAMPLE_RATE,
RealtimeEventCollector,
EventCollector,
build_realtime_url,
duplex_unit_boundary_ms,
read_pcm16_wav,
summarize_session_request_metrics,
)
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
from vllm_omni.clients.duplex import ( # noqa: E402
reference_audio_data_url as _ref_audio_data_url,
)
from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402
concat_frames_b64,
unit_subframe_offsets,
)
from vllm_omni.metrics.definitions import compute_audio_rtf # noqa: E402
def _url_with_model(url, model, *, autostart=None, session_id=None):
# This driver speaks the MiniCPM-o native duplex wire contract.
return build_realtime_url(
url,
model,
autostart=autostart,
session_id=session_id,
extra_query={"native_duplex": "1"},
)
_url_with_model = build_realtime_url
_read_wav_pcm16 = read_pcm16_wav
DemoArgs = argparse.Namespace | SimpleNamespace
def _audio_rtf_from_raw_metric(raw_metric: dict[str, object]) -> float | None:
"""Derive audio RTF from the raw timings reported by the client."""
generation_ms = raw_metric.get("audio_generation_ms")
duration_ms = raw_metric.get("audio_duration_ms")
if not isinstance(generation_ms, int | float) or not isinstance(duration_ms, int | float) or duration_ms <= 0:
return None
return round(compute_audio_rtf(float(generation_ms) / 1000.0, float(duration_ms) / 1000.0), 6)
def _video_frames_from_file(
path: Path,
*,
@@ -156,7 +177,7 @@ def _resolve_video_frames(args: DemoArgs) -> tuple[list[str], list[str | None]]:
@dataclass
class DemoState:
events: list[dict[str, object]] = field(default_factory=list)
timing_events: RealtimeEventCollector = field(default_factory=RealtimeEventCollector)
timing_events: EventCollector = field(default_factory=EventCollector)
audio_deltas: list[bytes] = field(default_factory=list)
response_audio_deltas: dict[str, list[bytes]] = field(default_factory=dict)
response_ids: list[str] = field(default_factory=list)
@@ -273,6 +294,11 @@ class DemoState:
after_s=created_at_s,
input_committed_at_s=input_committed_at_s,
response_id=response_id,
measurement_origin={
"ttft": "input_audio_buffer.commit client send to first non-empty text delta",
"ttfp": "input_audio_buffer.commit client send to first audio packet",
"rtf": "commit-to-last-audio receive time divided by emitted audio duration",
},
)
timing["measurement_origin"] = {
"response": "response.created client receive",
@@ -292,14 +318,14 @@ class DemoState:
metrics = timing.get("request_metrics")
if not isinstance(metrics, dict):
continue
requests.append(
{
"session_id": session_id,
"request_index": request_index,
"response_id": response_id,
**metrics,
}
)
request_metrics = {
"session_id": session_id,
"request_index": request_index,
"response_id": response_id,
**metrics,
"rtf": _audio_rtf_from_raw_metric(metrics),
}
requests.append(request_metrics)
return requests
def session_metric_summary(
@@ -602,7 +628,7 @@ def _session_update_event(args: DemoArgs) -> dict[str, object]:
"playback_commit_policy": "ack_only",
"extra_body": {
"auto_response": True,
"minicpmo45_native_duplex": True,
"native_duplex": True,
"force_listen_count": 0,
},
}
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Strict PersonaPlex validation for the unified OpenAI Realtime duplex path."""
from __future__ import annotations
@@ -18,17 +18,72 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import numpy as np
from vllm_omni.experimental.fullduplex.client import (
RealtimeDuplexClient,
wait_for,
try:
import websockets
from websockets.exceptions import ConnectionClosed
except ImportError as exc: # pragma: no cover - driver dependency.
raise SystemExit("Install websockets first: pip install websockets") from exc
from vllm_omni.clients.duplex import (
EventCollector,
write_pcm16_wav,
)
from vllm_omni.clients.duplex import (
wait_for_condition as wait_for,
)
SAMPLE_RATE_HZ = 24_000
FRAME_SAMPLES = 1_920
FRAME_PERIOD_S = FRAME_SAMPLES / SAMPLE_RATE_HZ
class RawRealtimeProbe:
"""Raw-event Realtime probe: one websocket plus an :class:`EventCollector`.
This driver speaks the wire protocol directly (it validates handshake
errors such as admission limits, which the public
:class:`vllm_omni.clients.duplex.DuplexClient` treats as failures), so it
keeps a transport-only client and reads everything from ``.events``.
"""
def __init__(self, url: str, *, max_size: int = 64 * 1024 * 1024) -> None:
self.url = url
self.max_size = max_size
self.events = EventCollector()
self._ws = None
self._reader_task: asyncio.Task[None] | None = None
async def __aenter__(self) -> RawRealtimeProbe:
self._ws = await websockets.connect(self.url, max_size=self.max_size)
self._reader_task = asyncio.create_task(self._read_events())
return self
async def __aexit__(self, exc_type, exc, traceback) -> None:
if self._ws is not None:
await self._ws.close()
if self._reader_task is not None:
self._reader_task.cancel()
try:
await self._reader_task
except asyncio.CancelledError:
pass
async def _read_events(self) -> None:
try:
while True:
raw = await self._ws.recv()
if not isinstance(raw, str):
continue
event = json.loads(raw)
if isinstance(event, dict):
self.events.add(event)
except ConnectionClosed:
return
async def send(self, event: dict[str, object]) -> None:
await self._ws.send(json.dumps(event))
def _input_identity(
path: Path,
*,
@@ -83,7 +138,7 @@ def _read_wav_as_float32(path: Path) -> np.ndarray:
return pcm
def _events(client: RealtimeDuplexClient, event_type: str) -> list[dict[str, object]]:
def _events(client: RawRealtimeProbe, event_type: str) -> list[dict[str, object]]:
return [event for event in client.events.events if event.get("type") == event_type]
@@ -93,8 +148,8 @@ async def _open_session(
session_id: str,
persona: str,
expect_error: bool = False,
) -> tuple[RealtimeDuplexClient, dict[str, object]]:
client = RealtimeDuplexClient(_realtime_url(args.url, args.model, session_id))
) -> tuple[RawRealtimeProbe, dict[str, object]]:
client = RawRealtimeProbe(_realtime_url(args.url, args.model, session_id))
await client.__aenter__()
await client.send(
{
@@ -121,7 +176,7 @@ async def _open_session(
return client, _events(client, event_type)[-1]
async def _close_session(client: RealtimeDuplexClient, *, timeout_s: float) -> None:
async def _close_session(client: RawRealtimeProbe, *, timeout_s: float) -> None:
await client.send({"type": "session.close"})
await wait_for(
lambda: client.events.count("session.closed") > 0,
@@ -132,7 +187,7 @@ async def _close_session(client: RealtimeDuplexClient, *, timeout_s: float) -> N
async def _stream_frames(
client: RealtimeDuplexClient,
client: RawRealtimeProbe,
pcm: np.ndarray,
*,
max_frames: int | None = None,
@@ -186,7 +241,7 @@ def _audio_frame_stats(
}
def _response_ids(client: RealtimeDuplexClient) -> set[str]:
def _response_ids(client: RawRealtimeProbe) -> set[str]:
return {
str(event["response_id"])
for event in _events(client, "response.audio.delta")
@@ -195,7 +250,7 @@ def _response_ids(client: RealtimeDuplexClient) -> set[str]:
def _session_result(
client: RealtimeDuplexClient,
client: RawRealtimeProbe,
*,
input_frames: int,
args: argparse.Namespace,
@@ -268,7 +323,7 @@ def _capabilities(created: dict[str, object]) -> dict[str, object]:
return capabilities
def _save(output_dir: Path, name: str, client: RealtimeDuplexClient, raw: bytes) -> None:
def _save(output_dir: Path, name: str, client: RawRealtimeProbe, raw: bytes) -> None:
(output_dir / f"{name}-events.jsonl").write_text(
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events),
encoding="utf-8",
@@ -228,7 +228,7 @@ async def _open_admission_session(
"session_id": session_id,
"model": args.model,
"modalities": ["audio", "text"],
"extra_body": {"minicpmo45_native_duplex": True},
"extra_body": {"native_duplex": True},
**({"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) else {}),
},
}
@@ -271,7 +271,7 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str,
"session_id": overflow_id,
"model": args.model,
"modalities": ["audio", "text"],
"extra_body": {"minicpmo45_native_duplex": True},
"extra_body": {"native_duplex": True},
**(
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
if getattr(args, "ref_audio", None)
@@ -337,7 +337,7 @@ async def _resume_probe(
"session_id": session_id,
"model": args.model,
"modalities": ["audio", "text"],
"extra_body": {"minicpmo45_native_duplex": True},
"extra_body": {"native_duplex": True},
**(
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
if getattr(args, "ref_audio", None)
@@ -442,7 +442,7 @@ async def _takeover_probe(
"session_id": session_id,
"model": args.model,
"modalities": ["audio", "text"],
"extra_body": {"minicpmo45_native_duplex": True},
"extra_body": {"native_duplex": True},
**(
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
if getattr(args, "ref_audio", None)
@@ -0,0 +1,139 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Live E2E coverage for the public duplex client (``vllm_omni.clients.duplex``).
Drives a real MiniCPM-o 4.5 duplex session end to end through
:class:`DuplexClient`: session handshake with reference audio, paced PCM
streaming, a speak response consumed via :class:`ResponseHandle` with
incremental playback acks, a forced transport drop recovered by automatic
``session.resume``, and a clean close.
"""
from __future__ import annotations
import asyncio
import uuid
from pathlib import Path
import pytest
from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
CORE_SERVER_PARAMS,
realtime_url,
resolve_ref_audio,
validated_input_wav,
)
from tests.helpers.mark import hardware_test
from vllm_omni.clients.duplex import (
DuplexClient,
EventCollector,
ReconnectPolicy,
audio_data_url,
read_pcm16_wav,
)
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
pytestmark = pytest.mark.omni
_SESSION_TIMEOUT_S = 900.0
_MAX_LISTEN_ROUNDS = 30
async def _run_live_session(*, url: str, model: str, ref_audio: Path, input_wav: Path) -> dict[str, object]:
# temperature 0.0 keeps the response-required reply short and
# deterministic (same as the validated demo driver).
config = create_duplex_session_config(ref_audio=audio_data_url(ref_audio), temperature=0.0)
client = DuplexClient(
url,
model=model,
config=config,
session_id=f"duplex-client-live-{uuid.uuid4().hex}",
reconnect=ReconnectPolicy(max_attempts=5, backoff_s=(0.25, 1.0)),
# Short interval so the periodic heartbeat path runs during the
# session — including across the forced transport drop below.
heartbeat_interval_s=5.0,
)
collector = EventCollector()
pcm = read_pcm16_wav(input_wav)
summary: dict[str, object] = {}
async with client:
collector_task = asyncio.create_task(collector.consume(client))
summary["session_id"] = client.session_id
summary["resume_token_issued"] = client.resume_token is not None
# Phase 1: stream the validated utterance and consume one speak response.
await client.stream_pcm(pcm, chunk_ms=200)
await client.commit()
listens = 0
async for response in client.responses():
if response.decision == "listen":
listens += 1
if listens > _MAX_LISTEN_ROUNDS:
raise AssertionError(f"no speak response after {listens} listen decisions")
continue
chunk_count = 0
audio_byte_count = 0
async for chunk in response.audio():
chunk_count += 1
audio_byte_count += len(chunk)
await client.ack_playback(response.played_ms)
summary.update(
{
"decision": response.decision,
"listen_rounds": listens,
"audio_chunks": chunk_count,
"audio_bytes": audio_byte_count,
"played_ms": response.played_ms,
"transcript": response.transcript,
}
)
break
# Phase 2: force-drop the transport underneath the client and require
# the automatic session.resume path to bring the session back.
resumed_waiter = asyncio.create_task(client.wait_for("session.resumed", timeout_s=90.0))
await asyncio.sleep(0.1) # let the waiter subscribe before the drop
assert client._ws is not None
await client._ws.close()
resumed = await resumed_waiter
summary["resumed_session_id"] = resumed.session_id or client.session_id
# The resumed transport must be live: a heartbeat gets acknowledged.
ack_waiter = asyncio.create_task(client.wait_for("session.heartbeat_ack", timeout_s=30.0))
await asyncio.sleep(0.1)
await client.send({"type": "session.heartbeat"})
await ack_waiter
summary["post_resume_heartbeat_ok"] = True
await client.close()
await asyncio.wait_for(collector_task, timeout=10.0)
summary["error_events"] = collector.errors()
summary["closed_cleanly"] = collector.count("session.closed") > 0
summary["timing"] = collector.timing_summary(after_s=0.0)
return summary
@pytest.mark.core_model
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", CORE_SERVER_PARAMS, indirect=True)
def test_duplex_client_live_session(omni_server) -> None:
summary = asyncio.run(
asyncio.wait_for(
_run_live_session(
url=realtime_url(omni_server),
model=omni_server.model,
ref_audio=resolve_ref_audio(),
input_wav=validated_input_wav(),
),
timeout=_SESSION_TIMEOUT_S,
)
)
assert summary["decision"] == "speak", summary
assert summary["audio_chunks"] > 0, summary
assert summary["played_ms"] > 0, summary
assert summary["transcript"], summary
assert summary["resume_token_issued"] is True, summary
assert summary["post_resume_heartbeat_ok"] is True, summary
assert summary["closed_cleanly"] is True, summary
assert summary["error_events"] == [], summary
@@ -30,7 +30,7 @@ from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import
run_multi_session,
)
from tests.helpers.mark import hardware_test
from vllm_omni.experimental.fullduplex.client import build_realtime_url
from vllm_omni.clients.duplex import build_realtime_url
from vllm_omni.experimental.fullduplex.video_stacking import concat_frames_b64
pytestmark = pytest.mark.omni
@@ -88,7 +88,13 @@ async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: f
async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[dict[str, object]]:
session_id = f"duplex-ci-protocol-{uuid.uuid4().hex}"
websocket_url = build_realtime_url(url, model, autostart=False, session_id=session_id)
websocket_url = build_realtime_url(
url,
model,
autostart=False,
session_id=session_id,
extra_query={"native_duplex": "1"},
)
async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws:
await ws.send(
json.dumps(
@@ -99,7 +105,7 @@ async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[
"model": model,
"modalities": ["audio", "text"],
"ref_audio": _ref_audio_data_url(str(ref_audio)),
"extra_body": {"minicpmo45_native_duplex": True},
"extra_body": {"native_duplex": True},
},
}
)
@@ -1001,7 +1001,7 @@ def test_realtime_duplex_demo_explicitly_enables_native_runtime_before_connect()
)
query = parse_qs(urlsplit(url).query)
assert query["minicpmo45_native_duplex"] == ["1"]
assert query["native_duplex"] == ["1"]
def test_realtime_duplex_demo_explicit_session_id_reaches_autostart_query():
@@ -1032,7 +1032,7 @@ def test_realtime_duplex_demo_session_update_uses_explicit_session_id():
assert event["type"] == "session.update"
assert "session_id" not in event
assert event["session"]["session_id"] == "reopen-e2e"
assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True
assert event["session"]["extra_body"]["native_duplex"] is True
def test_realtime_duplex_demo_response_required_uses_deterministic_sampling():
@@ -1,16 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_control_client import (
from vllm_omni.engine.duplex.control_client import (
DuplexControlClient,
DuplexControlRequestError,
)
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity
from vllm_omni.experimental.fullduplex.engine.messages import (
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
from vllm_omni.engine.duplex.messages import (
DuplexControlError,
DuplexControlResultMessage,
DuplexFence,
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -10,7 +10,7 @@ from dataclasses import FrozenInstanceError
import msgspec
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import (
from vllm_omni.engine.duplex.control_plane import (
DuplexControlPlane,
DuplexOutputContext,
DuplexRequestIdentity,
@@ -18,13 +18,8 @@ from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import (
DuplexStageSubmission,
DuplexStageSubmissionResult,
)
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
DuplexAppendPlan,
DuplexInputMode,
DuplexRuntimeCapabilities,
)
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity, DuplexLeaseConfig
from vllm_omni.experimental.fullduplex.engine.messages import (
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity, DuplexLeaseConfig
from vllm_omni.engine.duplex.messages import (
AppendDuplexInputMessage,
CloseDuplexSessionMessage,
DuplexFence,
@@ -34,6 +29,11 @@ from vllm_omni.experimental.fullduplex.engine.messages import (
SignalDuplexTurnMessage,
TouchDuplexSessionMessage,
)
from vllm_omni.engine.duplex.runtime import (
DuplexAppendPlan,
DuplexInputMode,
DuplexRuntimeCapabilities,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Focused tests for typed deployment-only runtime configuration."""
from __future__ import annotations
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key
from vllm_omni.engine.duplex.intermediate import get_stream_request_key
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -7,13 +7,13 @@ from dataclasses import FrozenInstanceError
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import DuplexInputMode
from vllm_omni.experimental.fullduplex.engine.duplex_session import DuplexSessionRuntimeManager
from vllm_omni.experimental.fullduplex.engine.lease import (
from vllm_omni.engine.duplex.lease import (
DuplexLeaseActivity,
DuplexLeaseConfig,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.engine.duplex.runtime import DuplexInputMode
from vllm_omni.engine.duplex.session import DuplexSessionRuntimeManager
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,24 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from dataclasses import FrozenInstanceError
from types import SimpleNamespace
import pytest
from vllm.sampling_params import SamplingParams
from vllm_omni.experimental.fullduplex.engine import duplex_runtime
from vllm_omni.experimental.fullduplex.engine.contracts import (
from vllm_omni.engine.duplex import runtime as duplex_runtime
from vllm_omni.engine.duplex.contracts import (
duplex_data_plane_request_info,
duplex_resource_request_belongs_to_session,
duplex_resource_request_id,
)
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.engine.duplex.runtime import (
DuplexInputMode,
DuplexOutputAction,
DuplexOutputDecision,
DuplexRuntimeCapabilities,
DuplexSessionRuntimeManager,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.minicpmo45.runtime import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import (
MiniCPMO45DuplexRuntimeExtension,
build_duplex_data_plane_prompt,
duplex_scheduler_token_budget,
+13 -10
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for AsyncOmniEngine.try_get_output and try_get_output_async.
Focuses on the critical behavior: when the orchestrator thread dies,
@@ -15,22 +18,22 @@ from pytest_mock import MockerFixture
from vllm_omni.engine.async_engine_utils import weak_shutdown_async_omni_engine
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.engine.duplex.control_client import (
DuplexControlClient,
DuplexControlRequestError,
)
from vllm_omni.engine.duplex.messages import (
AppendDuplexInputMessage,
DuplexControlResultMessage,
DuplexFence,
SignalDuplexTurnMessage,
)
from vllm_omni.engine.messages import (
CollectiveRPCResultMessage,
ErrorMessage,
OutputMessage,
)
from vllm_omni.engine.rpc_result_router import CorrelatedRpcClient
from vllm_omni.experimental.fullduplex.engine.duplex_control_client import (
DuplexControlClient,
DuplexControlRequestError,
)
from vllm_omni.experimental.fullduplex.engine.messages import (
AppendDuplexInputMessage,
DuplexControlResultMessage,
DuplexFence,
SignalDuplexTurnMessage,
)
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+107 -14
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -29,7 +29,9 @@ def _assert_isolated_import_succeeds(script: str) -> None:
env=env,
capture_output=True,
text=True,
timeout=60,
# The CLI probe imports the whole vllm + benchmark graph; on a loaded
# shared host that alone can take over a minute.
timeout=180,
check=False,
)
assert result.returncode == 0, result.stderr
@@ -43,7 +45,11 @@ def test_isolated_import_ignores_conflicting_gpu_visibility(monkeypatch: pytest.
_assert_isolated_import_succeeds("import vllm_omni.outputs")
def test_stable_engine_imports_do_not_load_experimental_duplex() -> None:
def test_stable_engine_imports_load_duplex_kernel_eagerly() -> None:
# The duplex kernel is imported eagerly by the stable engine modules.
# Model-specific duplex adapters must still load only via the
# dotted-string plugin paths, and the client-side vllm_omni.clients
# package must never enter the stable runtime import graph.
_assert_isolated_import_succeeds("""
import sys
@@ -51,14 +57,84 @@ import vllm_omni.engine.async_omni_engine
import vllm_omni.engine.orchestrator
import vllm_omni.entrypoints.async_omni
expected_eager = (
"vllm_omni.engine.duplex.contracts",
"vllm_omni.engine.duplex.control_plane",
"vllm_omni.entrypoints.duplex_request_client",
"vllm_omni.outputs.duplex",
)
missing = sorted(name for name in expected_eager if name not in sys.modules)
if missing:
raise SystemExit("duplex kernel modules not imported eagerly: " + ", ".join(missing))
forbidden_prefixes = (
"vllm_omni.clients",
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
"vllm_omni.model_executor.models.nemotron_voicechat.duplex",
"vllm_omni.model_executor.models.personaplex.duplex",
)
loaded = sorted(
name
for name in sys.modules
if name == "vllm_omni.experimental.fullduplex"
or name.startswith("vllm_omni.experimental.fullduplex.")
if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes)
)
if loaded:
raise SystemExit("stable imports loaded experimental duplex: " + ", ".join(loaded))
raise SystemExit("stable imports loaded plugin-only duplex modules: " + ", ".join(loaded))
""")
def test_cli_import_graph_stays_clear_of_the_client_package() -> None:
# `vllm-omni serve` imports the CLI package, whose __init__ eagerly pulls
# in the benchmark patch module and every benchmark subcommand. None of
# that may drag the client-side vllm_omni.clients package into the server
# process; the benchmark runtime imports it lazily at execution time.
_assert_isolated_import_succeeds("""
import sys
import vllm_omni.entrypoints.cli.main
loaded = sorted(
name
for name in sys.modules
if name == "vllm_omni.clients" or name.startswith("vllm_omni.clients.")
)
if loaded:
raise SystemExit("CLI import graph loaded the client package: " + ", ".join(loaded))
""")
def test_client_package_stays_clear_of_the_server_import_graph() -> None:
# vllm_omni.clients.duplex advertises a lightweight dependency set
# (pybase64 + websockets). Importing it and exercising the code paths
# that previously reached into the server tree (timing_summary once
# imported vllm_omni.metrics, dragging in prometheus_client) must not
# load any vllm_omni module outside the client package.
_assert_isolated_import_succeeds("""
import base64
import sys
import vllm_omni # the package __init__ loads its own baseline graph
baseline = set(sys.modules)
from vllm_omni.clients.duplex import EventCollector
collector = EventCollector()
chunk = base64.b64encode(b"\\x00\\x00" * 2400).decode("ascii")
collector.add({"type": "response.created", "response_id": "r1"}, received_at_s=1.0)
collector.add({"type": "response.audio.delta", "response_id": "r1", "delta": chunk}, received_at_s=1.1)
collector.add({"type": "response.audio.delta", "response_id": "r1", "delta": chunk}, received_at_s=1.2)
summary = collector.timing_summary(after_s=0.0, response_id="r1")
if "audio_output" not in summary or "request_metrics" not in summary:
raise SystemExit("timing_summary produced no audio_output/request_metrics sections")
added = sorted(
name
for name in set(sys.modules) - baseline
if name.startswith("vllm_omni.") and not name.startswith("vllm_omni.clients")
)
if added:
raise SystemExit("the client package pulled in server modules: " + ", ".join(added))
""")
@@ -95,26 +171,43 @@ if "duplex_output_decision" in fields:
""")
def test_stable_model_executor_does_not_expose_duplex_helper_module() -> None:
def test_duplex_sampling_helper_lives_in_model_executor() -> None:
_assert_isolated_import_succeeds("""
import importlib.util
if importlib.util.find_spec("vllm_omni.model_executor.duplex") is not None:
raise SystemExit("stable model_executor still exposes duplex helper module")
if importlib.util.find_spec("vllm_omni.model_executor.duplex_sampling") is None:
raise SystemExit("model_executor duplex_sampling helper module is missing")
""")
assert not (REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "model_executor.py").exists()
def test_runtime_package_does_not_bundle_the_browser_demo() -> None:
assert not (REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "web").exists()
def test_experimental_engine_uses_canonical_contract_module_names() -> None:
engine_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "engine"
def test_engine_duplex_uses_canonical_contract_module_names() -> None:
engine_dir = REPO_ROOT / "vllm_omni" / "engine" / "duplex"
core_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "core"
assert (engine_dir / "contracts.py").is_file()
assert (engine_dir / "lease.py").is_file()
assert (engine_dir / "messages.py").is_file()
for name in (
"contracts.py",
"lease.py",
"messages.py",
"session.py",
"control_plane.py",
"control_client.py",
"runtime.py",
"intermediate.py",
):
assert (engine_dir / name).is_file()
# the duplex_ prefix is dropped inside the duplex package
assert not (engine_dir / "duplex_session.py").exists()
assert not (engine_dir / "duplex_lease.py").exists()
assert not (engine_dir / "duplex_types.py").exists()
# the experimental package retains only the joyvl framework; stale
# __pycache__ leftovers may exist, so assert on source files, not the dir
old_engine_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "engine"
assert not list(old_engine_dir.glob("*.py"))
old_personaplex_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "personaplex"
assert not list(old_personaplex_dir.glob("**/*.py"))
assert not (core_dir / "identity.py").exists()
+15 -15
View File
@@ -21,6 +21,20 @@ from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs, FinishReason
from vllm.v1.engine.exceptions import EngineDeadError
from vllm.v1.metrics.stats import IterationStats
from vllm_omni.engine.duplex.control_plane import DuplexControlPlane
from vllm_omni.engine.duplex.messages import (
AppendDuplexInputMessage,
CloseDuplexSessionMessage,
DuplexFence,
OpenDuplexSessionMessage,
SignalDuplexTurnMessage,
)
from vllm_omni.engine.duplex.runtime import (
DuplexInputMode,
DuplexRuntimeCapabilities,
DuplexSessionRuntimeState,
duplex_resource_request_id,
)
from vllm_omni.engine.messages import (
AbortRequestMessage,
AbortResultMessage,
@@ -39,22 +53,8 @@ from vllm_omni.engine.orchestrator import (
_build_terminal_empty_output,
)
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.experimental.fullduplex.engine.duplex_control_plane import DuplexControlPlane
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
DuplexInputMode,
DuplexRuntimeCapabilities,
DuplexSessionRuntimeState,
duplex_resource_request_id,
)
from vllm_omni.experimental.fullduplex.engine.messages import (
AppendDuplexInputMessage,
CloseDuplexSessionMessage,
DuplexFence,
OpenDuplexSessionMessage,
SignalDuplexTurnMessage,
)
from vllm_omni.experimental.fullduplex.minicpmo45.runtime import MiniCPMO45DuplexRuntimeExtension
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import MiniCPMO45DuplexRuntimeExtension
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Streaming segment-boundary state must be keyed by stage.
``Orchestrator._orchestration_loop`` polls every stage into the same
@@ -15,13 +15,13 @@ from dataclasses import dataclass, field
import pytest
from vllm.sampling_params import SamplingParams
from vllm_omni.engine.duplex.contracts import DuplexRequestIdentity
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.engine.orchestrator import (
Orchestrator,
OrchestratorRequestState,
StreamingSegmentState,
)
from vllm_omni.experimental.fullduplex.engine.contracts import DuplexRequestIdentity
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -14,6 +14,11 @@ import pytest
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm_omni.engine.duplex.contracts import (
DuplexStageRequestContext,
DuplexStageSubmission,
)
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.engine.orchestrator import (
Orchestrator,
OrchestratorRequestState,
@@ -21,11 +26,6 @@ from vllm_omni.engine.orchestrator import (
_OrchestratorDuplexStagePort,
)
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.experimental.fullduplex.engine.contracts import (
DuplexStageRequestContext,
DuplexStageSubmission,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import base64
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.openai.audio import convert_input_audio_with_rate
from vllm_omni.entrypoints.duplex.audio import convert_input_audio_with_rate
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -10,7 +10,7 @@ from types import SimpleNamespace
import pytest
from vllm_omni.experimental.fullduplex.openai.runtime_adapter import (
from vllm_omni.entrypoints.duplex.runtime_adapter import (
validate_serving_runtime_adapter,
)
@@ -20,26 +20,33 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize(
"module_name",
[
"vllm_omni.experimental.fullduplex.openai.session_runner",
"vllm_omni.experimental.fullduplex.openai.runtime_bridge",
"vllm_omni.experimental.fullduplex.openai.serving",
"vllm_omni.entrypoints.duplex.session_runner",
"vllm_omni.entrypoints.duplex.runtime_bridge",
"vllm_omni.entrypoints.duplex.serving",
],
)
def test_generic_openai_runtime_import_does_not_load_minicpmo45(module_name: str) -> None:
def test_generic_openai_runtime_import_does_not_load_model_adapters(module_name: str) -> None:
repo_root = Path(__file__).resolve().parents[3]
script = f"""
import sys
import {module_name}
model_duplex_prefixes = (
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
"vllm_omni.model_executor.models.nemotron_voicechat.duplex",
"vllm_omni.model_executor.models.personaplex.duplex",
)
loaded = sorted(
name
for name in sys.modules
if name == "vllm_omni.experimental.fullduplex.minicpmo45"
or name.startswith("vllm_omni.experimental.fullduplex.minicpmo45.")
if any(
name == prefix or name.startswith(prefix + ".")
for prefix in model_duplex_prefixes
)
)
if loaded:
raise SystemExit("generic OpenAI runtime loaded MiniCPM-o: " + ", ".join(loaded))
raise SystemExit("generic OpenAI runtime loaded model duplex modules: " + ", ".join(loaded))
"""
result = subprocess.run(
@@ -59,7 +66,7 @@ def test_generic_handler_requires_explicit_serving_runtime_adapter() -> None:
script = """
from types import SimpleNamespace
from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
chat_service = SimpleNamespace(engine_client=SimpleNamespace())
try:
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Regression tests for duplex ``runtime_control`` payload redaction.
@@ -19,8 +19,8 @@ from enum import Enum
import pytest
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.openai.runtime_bridge import NativeRuntimeBridgeMixin
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.entrypoints.duplex.runtime_bridge import NativeRuntimeBridgeMixin
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
import pytest
from vllm_omni.experimental.fullduplex.openai.websocket import (
from vllm_omni.entrypoints.duplex.websocket import (
DuplexWebSocketActor,
normalize_duplex_input_event,
)
@@ -1,10 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
import json
import pytest
from vllm_omni.experimental.fullduplex.openai.protocol import (
DuplexCapabilities,
from vllm_omni.entrypoints.duplex.protocol import (
DuplexOverlapPolicy,
DuplexSession,
DuplexSessionConfig,
@@ -14,10 +16,13 @@ from vllm_omni.experimental.fullduplex.openai.protocol import (
DuplexTurnState,
ResponseCreateOptions,
)
from vllm_omni.experimental.fullduplex.openai.realtime_session import (
from vllm_omni.entrypoints.duplex.realtime_session import (
NativeRealtimeSessionProtocol,
)
from vllm_omni.experimental.fullduplex.openai.realtime_state import RealtimeStateOwner
from vllm_omni.entrypoints.duplex.realtime_state import RealtimeStateOwner
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
minicpmo45_native_capabilities,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -116,12 +121,28 @@ def test_realtime_model_name_does_not_implicitly_enable_native_duplex():
event = protocol._session_create_from_realtime({"model": "openbmb/MiniCPM-o-4_5"})
assert "native_duplex" not in event["session"]["extra_body"]
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
def test_realtime_explicit_native_duplex_flag_is_preserved():
protocol = NativeRealtimeSessionProtocol({})
event = protocol._session_create_from_realtime(
{
"model": "openbmb/MiniCPM-o-4_5",
"extra_body": {"native_duplex": True},
}
)
assert event["session"]["extra_body"]["native_duplex"] is True
def test_realtime_legacy_native_duplex_alias_is_normalized():
# The deprecated model-prefixed spelling still opts in, but everything
# downstream (and every session echo) sees only the canonical key.
protocol = NativeRealtimeSessionProtocol({})
event = protocol._session_create_from_realtime(
{
"model": "openbmb/MiniCPM-o-4_5",
@@ -129,10 +150,25 @@ def test_realtime_explicit_native_duplex_flag_is_preserved():
}
)
assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True
assert event["session"]["extra_body"]["native_duplex"] is True
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostart():
protocol = NativeRealtimeSessionProtocol(
{
"model": "openbmb/MiniCPM-o-4_5",
"native_duplex": "1",
}
)
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
assert event["type"] == "session.create"
assert event["session"]["extra_body"]["native_duplex"] is True
def test_realtime_legacy_query_native_duplex_alias_seeds_canonical_key():
protocol = NativeRealtimeSessionProtocol(
{
"model": "openbmb/MiniCPM-o-4_5",
@@ -143,7 +179,8 @@ def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostar
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
assert event["type"] == "session.create"
assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True
assert event["session"]["extra_body"]["native_duplex"] is True
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
@pytest.mark.asyncio
@@ -151,7 +188,7 @@ async def test_realtime_resume_heartbeat_and_event_ack_translate_without_session
protocol = NativeRealtimeSessionProtocol(
{
"model": "openbmb/MiniCPM-o-4_5",
"minicpmo45_native_duplex": "1",
"native_duplex": "1",
}
)
@@ -217,7 +254,7 @@ async def test_realtime_resume_query_suppresses_model_autostart_on_same_url():
protocol = NativeRealtimeSessionProtocol(
{
"model": "openbmb/MiniCPM-o-4_5",
"minicpmo45_native_duplex": "1",
"native_duplex": "1",
"resume": "1",
}
)
@@ -293,6 +330,39 @@ def test_realtime_projects_resume_lifecycle_events_without_duplex_prefix():
}
def test_realtime_projects_listen_response_identity():
protocol = NativeRealtimeSessionProtocol({})
protocol.encode_outbound_event({"type": "session.created", "session": {"id": "sid-listen"}})
bound = protocol.encode_outbound_event(
{
"type": "response.listen",
"session_id": "sid-listen",
"epoch": 0,
"reason": "model_listen",
"model_listen": True,
"response_id": "resp_1",
}
)[0]
# A listen that terminates a precreated response keeps the response
# identity on the Realtime dialect; clients demultiplex on response_id.
assert bound["type"] == "response.listen"
assert bound["response_id"] == "resp_1"
assert bound["response"]["id"] == "resp_1"
assert bound["response"]["status"] == "listening"
anonymous = protocol.encode_outbound_event(
{
"type": "response.listen",
"session_id": "sid-listen",
"epoch": 0,
"reason": "silence_or_noise",
}
)[0]
assert "response_id" not in anonymous
assert "id" not in anonymous["response"]
def test_duplex_session_registry_advances_incarnation_when_id_is_reused():
registry = DuplexSessionRegistry()
@@ -436,7 +506,7 @@ def test_duplex_capabilities_do_not_claim_core_kv_or_input_append():
def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease():
caps = DuplexCapabilities.minicpmo45_native(max_sessions=2).as_dict()
caps = minicpmo45_native_capabilities(max_sessions=2).as_dict()
assert caps["implementation_level"] == "model_native_duplex"
assert caps["supports_input_append"] is True
@@ -463,7 +533,7 @@ def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease():
def test_duplex_overlap_policy_auto_falls_back_to_listen_only():
from vllm_omni.experimental.fullduplex.openai.protocol import (
from vllm_omni.entrypoints.duplex.protocol import (
DuplexOverlapPolicy,
DuplexSessionConfig,
)
@@ -472,7 +542,7 @@ def test_duplex_overlap_policy_auto_falls_back_to_listen_only():
def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment():
caps = DuplexCapabilities.minicpmo45_native(max_sessions=1).as_dict()
caps = minicpmo45_native_capabilities(max_sessions=1).as_dict()
assert caps["supports_multi_session"] is False
assert caps["supports_multi_session_same_replica"] is False
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -9,7 +9,7 @@ import json
import pytest
from vllm_omni.experimental.fullduplex.openai.session_attachment import (
from vllm_omni.entrypoints.duplex.session_attachment import (
DuplexEventJournal,
DuplexJournalGapError,
DuplexJournalOverflowError,
@@ -43,7 +43,7 @@ def test_resume_credential_uses_256_bit_token_digest_and_redacted_repr(monkeypat
credential = DuplexResumeCredential.from_token(token)
calls = []
import vllm_omni.experimental.fullduplex.openai.session_attachment as attachment_module
import vllm_omni.entrypoints.duplex.session_attachment as attachment_module
original_compare = attachment_module.hmac.compare_digest
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import ast
@@ -7,13 +10,37 @@ import pytest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_REPO_ROOT = Path(__file__).parents[3]
_CLIENTS_PACKAGE = "vllm_omni.clients"
def test_api_server_does_not_import_experimental_duplex_at_module_load() -> None:
api_server = Path(__file__).parents[3] / "vllm_omni/entrypoints/openai/api_server.py"
module = ast.parse(api_server.read_text())
top_level_imports = {
node.module for node in module.body if isinstance(node, ast.ImportFrom) and isinstance(node.module, str)
}
def _imports_clients_package(path: Path) -> bool:
"""True if the module imports the client-side package at module level.
assert not any(module.startswith("vllm_omni.experimental.fullduplex") for module in top_level_imports)
``vllm_omni.clients`` is client-side by construction: the serving stack
(imported eagerly by the API server) must never depend on it.
"""
module = ast.parse(path.read_text(encoding="utf-8"))
for node in module.body:
if isinstance(node, ast.Import):
if any(
alias.name == _CLIENTS_PACKAGE or alias.name.startswith(_CLIENTS_PACKAGE + ".") for alias in node.names
):
return True
elif isinstance(node, ast.ImportFrom):
if node.module and (node.module == _CLIENTS_PACKAGE or node.module.startswith(_CLIENTS_PACKAGE + ".")):
return True
if node.module == "vllm_omni" and any(alias.name == "clients" for alias in node.names):
return True
return False
def test_api_server_does_not_import_client_package() -> None:
api_server = _REPO_ROOT / "vllm_omni/entrypoints/openai/api_server.py"
assert not _imports_clients_package(api_server)
def test_duplex_serving_stack_does_not_import_client_package() -> None:
duplex_pkg = _REPO_ROOT / "vllm_omni/entrypoints/duplex"
offenders = [path.name for path in sorted(duplex_pkg.glob("*.py")) if _imports_clients_package(path)]
assert not offenders, f"duplex serving modules import the client package: {offenders}"
@@ -14,46 +14,59 @@ import numpy as np
import pytest
from starlette.websockets import WebSocketDisconnect
# Import the OpenAI entrypoints package at collection time, before pytest
# attaches its caplog handlers to the root logger. vllm's api_server
# transitively imports model_hosting_container_standards, whose import-time
# configure_root_logger() raises the level of every handler then attached to
# the root logger to ERROR. The serving runtime under test reaches this chain
# lazily (runtime_bridge -> entrypoints.openai.protocol.audio), so without
# this eager import the side effect fires mid-test and silences caplog for
# the remainder of the session. Before the capability predicate moved to
# vllm_omni.entrypoints.duplex.capability, this file imported the package at
# module scope implicitly.
import vllm_omni.entrypoints.openai # noqa: F401
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlRequestError
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence, DuplexSessionLifecycleMessage
from vllm_omni.experimental.fullduplex.minicpmo45 import (
MiniCPMO45NativeDuplexServingAdapter,
MiniCPMO45PcmAppendBuffer,
)
from vllm_omni.experimental.fullduplex.minicpmo45.data_plane import (
MiniCPMO45DataPlaneContext,
MiniCPMO45DataPlaneSession,
)
from vllm_omni.experimental.fullduplex.minicpmo45.runtime import (
MiniCPMO45DuplexRuntimeExtension,
)
from vllm_omni.experimental.fullduplex.minicpmo45.session import (
MiniCPMO45ServingSessionState,
)
from vllm_omni.experimental.fullduplex.openai import vad as realtime_vad
from vllm_omni.experimental.fullduplex.openai.protocol import (
DuplexCapabilities,
from vllm_omni.engine.duplex.control_client import DuplexControlRequestError
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
from vllm_omni.engine.duplex.messages import DuplexFence, DuplexSessionLifecycleMessage
from vllm_omni.engine.duplex.runtime import duplex_resource_request_id
from vllm_omni.entrypoints.duplex import vad as realtime_vad
from vllm_omni.entrypoints.duplex.protocol import (
DuplexOverlapPolicy,
DuplexPlaybackCommitPolicy,
DuplexSession,
DuplexSessionConfig,
ResponseCreateOptions,
)
from vllm_omni.experimental.fullduplex.openai.realtime_session import NativeRealtimeSessionProtocol
from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ServingRuntimeConfigError
from vllm_omni.experimental.fullduplex.openai.serving import (
from vllm_omni.entrypoints.duplex.realtime_session import NativeRealtimeSessionProtocol
from vllm_omni.entrypoints.duplex.runtime_adapter import ServingRuntimeConfigError
from vllm_omni.entrypoints.duplex.serving import (
OmniDuplexSessionHandler,
should_enable_duplex_endpoint,
)
from vllm_omni.experimental.fullduplex.openai.websocket import DuplexWebSocketActor
from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision
from vllm_omni.experimental.fullduplex.personaplex.serving_adapter import (
from vllm_omni.entrypoints.duplex.websocket import DuplexWebSocketActor
from vllm_omni.model_executor.models.minicpmo_4_5.duplex import (
MiniCPMO45NativeDuplexServingAdapter,
MiniCPMO45PcmAppendBuffer,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
minicpmo45_native_capabilities,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import (
MiniCPMO45DataPlaneContext,
MiniCPMO45DataPlaneSession,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import (
MiniCPMO45DuplexRuntimeExtension,
)
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import (
MiniCPMO45ServingSessionState,
)
from vllm_omni.model_executor.models.personaplex.duplex.serving_adapter import (
PersonaPlexServingRuntimeAdapter,
)
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.outputs.duplex import attach_duplex_output_decision
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -62,12 +75,12 @@ def test_native_input_append_supports_explicit_session_opt_out():
session = DuplexSession(
"sid-explicit-tts",
DuplexSessionConfig(model="test-model"),
capabilities=DuplexCapabilities.minicpmo45_native(),
capabilities=minicpmo45_native_capabilities(),
)
assert OmniDuplexSessionHandler._uses_native_input_append(session) is True
session.config.extra_body["minicpmo45_native_duplex"] = False
session.config.extra_body["native_duplex"] = False
assert OmniDuplexSessionHandler._uses_native_input_append(session) is False
@@ -293,7 +306,7 @@ class FakeEngineClient:
class FakeChatService:
duplex_serving_adapter_path = (
"vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
)
def __init__(self, engine_client: FakeEngineClient) -> None:
@@ -844,7 +857,7 @@ def test_native_response_options_ignore_private_runtime_config():
session = DuplexSession(
session_id="sid-response-private-config",
config=DuplexSessionConfig(),
capabilities=DuplexCapabilities.minicpmo45_native(),
capabilities=minicpmo45_native_capabilities(),
)
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
@@ -878,7 +891,7 @@ def test_native_response_create_rejects_unwired_generation_options(payload):
session = DuplexSession(
session_id="sid-response-unwired-options",
config=DuplexSessionConfig(),
capabilities=DuplexCapabilities.minicpmo45_native(),
capabilities=minicpmo45_native_capabilities(),
)
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
@@ -1244,7 +1257,7 @@ async def test_minicpmo_native_session_update_rejects_native_duplex_mode_flip():
{
"type": "turn.signal",
"event": "session.update",
"payload": {"extra_body": {"minicpmo45_native_duplex": False}},
"payload": {"extra_body": {"native_duplex": False}},
}
)
ws.put(
@@ -1581,7 +1594,7 @@ def _native_session_create(
event["session"]["model"] = "openbmb/MiniCPM-o-4_5"
event["session"]["modalities"] = list(modalities or ["text"])
event["session"]["instructions"] = "You are a concise assistant."
event["session"]["extra_body"] = {"minicpmo45_native_duplex": True}
event["session"]["extra_body"] = {"native_duplex": True}
return event
@@ -1598,7 +1611,7 @@ def _native_realtime_session_update(
"modalities": list(modalities or ["text"]),
"instructions": "You are a concise assistant.",
"idle_timeout_s": 1,
"extra_body": {"minicpmo45_native_duplex": True},
"extra_body": {"native_duplex": True},
},
}
@@ -1674,7 +1687,7 @@ def _install_direct_silence_scheduler(
handler: OmniDuplexSessionHandler,
session: DuplexSession,
) -> None:
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
async def _schedule(payload: object, **kwargs: Any) -> bool:
if handler._native_silence_continuation_is_stale(
@@ -1819,7 +1832,7 @@ def test_legacy_auto_overlap_policy_falls_back_to_listen_only():
overlap_policy=DuplexSessionConfig._normalize_overlap_policy("auto"),
),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
decision = handler._overlap_decision(
session,
@@ -2066,11 +2079,11 @@ def test_auto_response_post_response_silence_remains_model_owned_input():
@pytest.mark.asyncio
async def test_auto_response_terminal_advances_open_realtime_input_without_commit():
handler, session = _auto_response_context("sid-auto-realtime-open-input", playback_active=True)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
response_id = session.active_response_id
assert response_id is not None
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
native.input_since_commit = True
native.speech_since_commit = True
session.accumulate_overlap_speech(800)
@@ -2113,11 +2126,11 @@ async def test_auto_response_terminal_allows_next_model_unit_while_playback_drai
"sid-auto-realtime-open-before-overlap",
playback_active=True,
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
response_id = session.active_response_id
assert response_id is not None
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
native.input_since_commit = True
native.speech_since_commit = True
session.end_response(commit_text=False, preserve_request=True)
@@ -2147,11 +2160,11 @@ async def test_auto_response_terminal_allows_next_model_unit_while_playback_drai
@pytest.mark.asyncio
async def test_realtime_committed_overlap_promotes_after_response_done():
handler, session = _auto_response_context("sid-auto-realtime-committed-input", playback_active=True)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
response_id = session.active_response_id
assert response_id is not None
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
native.committed_audio_payload = _native_audio_payload()
native.deferred_response_create = True
session.accumulate_overlap_speech(1000)
@@ -3061,7 +3074,7 @@ async def test_native_append_propagates_current_turn_fence_to_engine():
session_id="sid-fenced-append",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.turn_id = 2
session.begin_response(turn_id=2)
ws = TimedWebSocket()
@@ -3091,7 +3104,7 @@ async def test_minicpmo_auto_response_tts_segment_boundary_appends_silence_unit(
session_id="sid-segment-boundary",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
@@ -3140,7 +3153,7 @@ async def test_minicpmo_auto_response_pre_response_tts_boundary_continues_model_
session_id="sid-pre-response-boundary",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
_install_direct_silence_scheduler(handler, session)
ws = TimedWebSocket()
@@ -3187,11 +3200,11 @@ async def test_minicpmo_pre_response_continuation_drops_after_model_turn_ends():
session_id="sid-stale-pre-response-boundary",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
model_turn_id = session.turn_id
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
async def _stale_before_append(payload: object, **kwargs: Any) -> bool:
session.complete_model_turn(model_turn_id)
@@ -3237,7 +3250,7 @@ async def test_minicpmo_auto_response_continuation_stops_at_large_safety_boundar
session_id="sid-long-response",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
@@ -3283,10 +3296,10 @@ async def test_minicpmo_auto_response_boundary_listen_closes_response():
session_id="sid-boundary-listen",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response(turn_id=0)
session.bind_request(request_id)
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
native.continuation_owner_id = f"response:{response_id}"
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
_install_direct_silence_scheduler(handler, session)
@@ -3313,6 +3326,11 @@ async def test_minicpmo_auto_response_boundary_listen_closes_response():
assert emitted is True
assert engine.appended == []
assert ws.sent_types() == ["response.listen", "response.done"]
# The listen terminates the precreated response: clients demultiplex the
# decision by its id, so the payload must carry the response identity.
listen_payload, done_payload = ws.sent[-2], ws.sent[-1]
assert listen_payload["response_id"] == response_id
assert done_payload["response_id"] == response_id
assert session.active_response_id is None
assert session.active_request_id == request_id
assert session.turn_id == 1
@@ -3327,10 +3345,10 @@ async def test_minicpmo_auto_response_boundary_allows_next_model_turn_response()
session_id="sid-boundary-next-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.bind_request(request_id)
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
native.continuation_owner_id = f"response:{session.active_response_id}"
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
ws = TimedWebSocket()
@@ -3382,10 +3400,10 @@ async def test_minicpmo_auto_response_boundary_does_not_close_new_epoch_response
session_id="sid-boundary-race",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
old_response_id = session.begin_response(turn_id=0)
session.bind_request(request_id)
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
native.continuation_owner_id = f"response:{old_response_id}"
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
sent: list[dict[str, Any]] = []
@@ -3418,7 +3436,7 @@ async def test_minicpmo_auto_response_pre_speak_listen_continues_same_response()
session_id="sid-pre-speak-listen",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
@@ -3458,7 +3476,7 @@ async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_req
session_id="sid-listen-before-response",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
ws = TimedWebSocket()
@@ -3482,7 +3500,7 @@ async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_req
assert close_reason is None
assert emitted is True
assert session.active_request_id == request_id
assert not handler._minicpmo_data_plane.is_terminal(request_id)
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
assert ws.sent_types().count("response.listen") == 1
@@ -3666,7 +3684,7 @@ async def test_minicpmo_auto_response_post_speak_listen_continues_same_response(
session_id="sid-post-speak-listen",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response(turn_id=0)
session.turn_id = 1
session.bind_request(request_id)
@@ -3707,7 +3725,7 @@ async def test_minicpmo_auto_response_turn_end_preserves_resumable_request():
session_id="sid-turn-end-preserve-request",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.begin_response(turn_id=0)
session.bind_request(request_id)
ws = TimedWebSocket()
@@ -3746,7 +3764,7 @@ async def test_minicpmo_auto_response_empty_turn_end_emits_model_listen():
session_id="sid-empty-turn-end",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
ws = TimedWebSocket()
@@ -3789,7 +3807,7 @@ async def test_minicpmo_auto_response_drops_late_audio_from_completed_model_turn
session_id="sid-late-completed-turn",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
session.complete_model_turn(0)
ws = TimedWebSocket()
@@ -4015,7 +4033,7 @@ async def test_duplex_auto_response_empty_terminal_does_not_create_empty_respons
)
native_results = list(
handler._minicpmo_data_plane.project(
handler._serving_runtime_adapter.data_plane.project(
{"data_plane_outputs": [output]},
context=_data_plane_context(session),
)
@@ -4030,7 +4048,7 @@ async def test_duplex_auto_response_empty_terminal_does_not_create_empty_respons
assert "response.created" not in [m.get("type") for m in sent]
assert "response.done" not in [m.get("type") for m in sent]
assert session.turn_id == 1
assert not handler._minicpmo_data_plane.is_terminal(request_id)
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
@pytest.mark.asyncio
@@ -4040,11 +4058,11 @@ async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_pl
engine = FakeEngineClient()
chat_service = FakeChatService(engine)
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
handler._minicpmo_data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32))
handler._minicpmo_data_plane.segment_text_delta(request_id, "hello")
handler._minicpmo_data_plane.mark_terminal(request_id)
handler._serving_runtime_adapter.data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32))
handler._serving_runtime_adapter.data_plane.segment_text_delta(request_id, "hello")
handler._serving_runtime_adapter.data_plane.mark_terminal(request_id)
native = MiniCPMO45ServingSessionState()
handler._minicpmo_sessions[session_id] = native
handler._serving_runtime_adapter.session_states[session_id] = native
event = _native_session_create(session_id)
event["session"]["extra_body"]["auto_response"] = True
@@ -4054,8 +4072,8 @@ async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_pl
await handler.handle_session(ws)
assert not handler._minicpmo_data_plane.has_request(request_id)
assert session_id not in handler._minicpmo_sessions
assert not handler._serving_runtime_adapter.data_plane.has_request(request_id)
assert session_id not in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
@@ -4923,7 +4941,7 @@ async def test_realtime_disconnect_detaches_resumable_session_without_runtime_cl
("sid-resumable-disconnect", DuplexLeaseActivity.DETACH),
]
assert handler._registry.get("sid-resumable-disconnect") is not None
assert "sid-resumable-disconnect" in handler._minicpmo_sessions
assert "sid-resumable-disconnect" in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
@@ -4947,7 +4965,7 @@ async def test_realtime_initial_resume_token_delivery_failure_closes_unrecoverab
assert engine.closed == [("sid-created-delivery-failure", "disconnect")]
assert handler._registry.get("sid-created-delivery-failure") is None
assert "sid-created-delivery-failure" not in handler._minicpmo_sessions
assert "sid-created-delivery-failure" not in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
@@ -4983,7 +5001,7 @@ async def test_realtime_send_after_created_disconnect_preserves_resumable_sessio
("sid-created-then-disconnected", DuplexLeaseActivity.DETACH),
]
assert handler._registry.get("sid-created-then-disconnected") is not None
assert "sid-created-then-disconnected" in handler._minicpmo_sessions
assert "sid-created-then-disconnected" in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
@@ -5007,7 +5025,7 @@ async def test_realtime_resume_rotates_token_replays_and_preserves_runtime_ident
second = TimedWebSocket(receive_timeout_s=0.1)
second.query_params = {
"model": "openbmb/MiniCPM-o-4_5",
"minicpmo45_native_duplex": "1",
"native_duplex": "1",
"resume": "1",
}
second.put(
@@ -5083,7 +5101,7 @@ async def test_realtime_resume_preserves_append_tail_order_across_connections():
assert created is not None
session = handler._registry.get("sid-resume-append-order")
assert session is not None
native_state = handler._minicpmo_session_state(session)
native_state = handler._runtime_session_state(session)
first_scheduler = native_state.silence_continuation_scheduler
assert first_scheduler is not None
first.put(
@@ -5101,7 +5119,7 @@ async def test_realtime_resume_preserves_append_tail_order_across_connections():
second = TimedWebSocket(receive_timeout_s=0.5)
second.query_params = {
"model": "openbmb/MiniCPM-o-4_5",
"minicpmo45_native_duplex": "1",
"native_duplex": "1",
"resume": "1",
}
second.put(
@@ -5402,7 +5420,7 @@ async def test_realtime_disconnect_grace_cancels_only_orphan_response():
assert engine.aborted == ["orphan-request"]
assert orphan_task.cancelled()
assert session.active_response_id is None
assert "sid-grace-response" in handler._minicpmo_sessions
assert "sid-grace-response" in handler._serving_runtime_adapter.session_states
@pytest.mark.asyncio
@@ -5453,7 +5471,7 @@ async def test_realtime_idle_ttl_lifecycle_removes_detached_serving_projection()
await asyncio.sleep(0.05)
assert handler._registry.get("sid-idle-expired") is None
assert "sid-idle-expired" not in handler._minicpmo_sessions
assert "sid-idle-expired" not in handler._serving_runtime_adapter.session_states
assert "sid-idle-expired" not in handler._session_tasks
assert engine.closed == []
@@ -6428,7 +6446,7 @@ async def test_minicpmo_native_duplex_separates_public_and_runtime_config(monkey
return [0.25] * 1600, 16000
monkeypatch.setattr(
"vllm_omni.experimental.fullduplex.minicpmo45.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio",
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio",
fake_resolve_ref_audio,
)
event = _native_session_create("sid-native-ref-audio", modalities=["text", "audio"])
@@ -6554,7 +6572,7 @@ async def test_minicpmo_native_duplex_preserves_ref_audio_channels_until_normali
return np.tile(np.array([[0.25, -0.25], [0.5, -0.5]], dtype=np.float32), (800, 1)), 16000
monkeypatch.setattr(
"vllm_omni.experimental.fullduplex.minicpmo45.adapter.MediaConnector",
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.adapter.MediaConnector",
FakeMediaConnector,
)
@@ -6580,7 +6598,7 @@ async def test_minicpmo_native_duplex_rejects_ref_audio_path():
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
event = _native_session_create("sid-native-ref-path")
event["session"]["extra_body"] = {
"minicpmo45_native_duplex": True,
"native_duplex": True,
"ref_audio_path": "/tmp/ref.wav",
}
ws = TimedWebSocket()
@@ -6603,7 +6621,7 @@ def test_minicpmo_native_duplex_explicit_barge_in_request_interrupts():
session = DuplexSession(
session_id="sid-native-explicit-barge-disabled",
config=DuplexSessionConfig(overlap_policy=DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value),
capabilities=DuplexCapabilities.minicpmo45_native(),
capabilities=minicpmo45_native_capabilities(),
)
payload = {
"type": "audio",
@@ -6767,9 +6785,9 @@ async def test_minicpmo_auto_response_restarts_drain_when_append_races_idle_exit
session_id="sid-native-late-output",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
session.bind_request(request_id)
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
projected_batches: list[object] = []
projected = asyncio.Event()
@@ -7521,7 +7539,7 @@ async def test_minicpmo_native_auto_response_preserves_silence_continuations_acr
session.capabilities.chunk_period_ms = 50
session.bind_request(request_id)
response_id = session.begin_response(turn_id=session.turn_id)
native = handler._minicpmo_session_state(session)
native = handler._runtime_session_state(session)
scheduler = native.silence_continuation_scheduler
assert scheduler is not None
@@ -7598,7 +7616,7 @@ async def test_minicpmo_native_duplex_uses_segment_text_metadata_for_transcript_
async def emit(output: object) -> None:
result = {"data_plane_outputs": [output]}
for native_result in handler._minicpmo_data_plane.project(
for native_result in handler._serving_runtime_adapter.data_plane.project(
result,
context=_data_plane_context(session),
):
@@ -7682,7 +7700,7 @@ async def test_minicpmo_native_duplex_continuous_speak_reuses_active_response_un
async def emit(output: object) -> None:
result = {"data_plane_outputs": [output]}
for native_result in handler._minicpmo_data_plane.project(
for native_result in handler._serving_runtime_adapter.data_plane.project(
result,
context=_data_plane_context(session),
):
@@ -7932,7 +7950,7 @@ async def test_minicpmo_auto_response_segment_complete_continues_until_model_lis
session_id="sid-native-segment-complete",
config=DuplexSessionConfig(extra_body={"auto_response": True}),
)
session.capabilities = DuplexCapabilities.minicpmo45_native()
session.capabilities = minicpmo45_native_capabilities()
response_id = session.begin_response()
request_id = "duplex-sid-native-segment-complete-stage0"
session.bind_request(request_id)
@@ -7968,7 +7986,7 @@ async def test_minicpmo_auto_response_segment_complete_continues_until_model_lis
assert emitted is False
assert session.active_response_id == response_id
assert session.active_request_id == request_id
assert not handler._minicpmo_data_plane.is_terminal(request_id)
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
assert sent == []
assert len(engine.appended) == 1
_, mode, payload, final = engine.appended[0]
+11 -8
View File
@@ -1,21 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
import pytest
from vllm_omni.engine.messages import OutputMessage
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.client_request_state import ClientRequestState
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.engine.duplex.runtime import (
DuplexOutputAction,
DuplexOutputDecision,
duplex_resource_request_id,
)
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision
from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient
from vllm_omni.engine.messages import OutputMessage
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.client_request_state import ClientRequestState
from vllm_omni.entrypoints.duplex_request_client import DuplexRequestClient
from vllm_omni.metrics.stats import OrchestratorAggregator, StageRequestStats, StageStats
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.outputs.duplex import attach_duplex_output_decision
def test_async_omni_uses_extracted_duplex_request_client():
@@ -1,10 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from tests.entrypoints.openai_api.test_duplex_handler import FakeChatService, TimedWebSocket
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSession, DuplexSessionConfig
from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler
from vllm_omni.entrypoints.duplex.protocol import DuplexSession, DuplexSessionConfig
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -175,21 +175,26 @@ def test_open_streaming_response_requires_post_commit_drain():
def test_final_input_commit_checkpoints_playback_first():
demo = _load_demo_module()
client = demo.RealtimeDuplexClient("ws://unused")
client.events.add(
collector = demo.EventCollector()
collector.add(
{
"type": "response.created",
"response": {"id": "resp-zero-audio"},
}
)
calls = []
calls: list[dict[str, object]] = []
async def send(event):
calls.append(event)
class _Client:
async def ack_playback(self, played_ms, *, response_id=None, item_id=None, committed_ms=None):
calls.append({"type": "playback.ack", "response_id": response_id, "played_ms": int(played_ms)})
client.send = send
commit_sent_at_s = asyncio.run(demo._commit_input_after_playback_checkpoint(client))
async def commit(self):
calls.append({"type": "input_audio_buffer.commit"})
commit_sent_at_s = asyncio.run(demo._commit_input_after_playback_checkpoint(_Client(), collector))
# The zero-audio response is still acked (0 ms) before the commit: the
# ack checkpoints its history position on the server.
assert [event["type"] for event in calls] == ["playback.ack", "input_audio_buffer.commit"]
assert calls[0]["response_id"] == "resp-zero-audio"
assert calls[0]["played_ms"] == 0
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import re
import shutil
import subprocess
@@ -33,7 +36,7 @@ def test_client_uses_proxy_relative_realtime_url_and_model_policy_session():
assert "new URL(config.realtimePath, window.location.href)" in source
assert "url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'" in source
assert "url.searchParams.set('autostart', '0')" in source
assert "url.searchParams.set('minicpmo45_native_duplex', '1')" in source
assert "url.searchParams.set('native_duplex', '1')" in source
assert "auto_response: true" in source
assert "input_audio_buffer.append" in source
assert "input_audio_buffer.commit" not in source
Whitespace-only changes.
@@ -1,8 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import pytest
from vllm_omni.experimental.fullduplex.openai.commit_policy import (
from vllm_omni.entrypoints.duplex.commit_policy import (
CommitAction,
CommitSnapshot,
decide_commit_action,
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import base64
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.minicpmo45.input import MiniCPMO45PcmAppendBuffer
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.input import MiniCPMO45PcmAppendBuffer
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
MiniCPMO45DuplexPolicy,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_seed_tts_initial_text_is_part_of_native_duplex_context():
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
"Speak exactly.",
True,
"The quick brown fox.",
)
assert prefix == "<|im_start|>system\nSpeak exactly.\n<|audio_start|>"
assert suffix == (
"<|audio_end|><|im_end|>\n<|im_start|>user\nThe quick brown fox.<|im_end|>\n<|im_start|>assistant\n"
)
@@ -61,7 +61,7 @@ class TestRegistryDeclaration:
pipeline = OMNI_PIPELINES[_PIPELINE_KEY]
assert pipeline.duplex_control_enabled is True
assert pipeline.duplex_serving_adapter == (
"vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
)
assert not hasattr(pipeline, "max_native_duplex_sessions")
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
from types import SimpleNamespace
@@ -5,9 +8,9 @@ from types import SimpleNamespace
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import (
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.engine.duplex.runtime import duplex_resource_request_id
from vllm_omni.model_executor.models.nemotron_voicechat.duplex.data_plane import (
NemotronVoiceChatDataPlaneContext,
NemotronVoiceChatDataPlaneSession,
)
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import base64
@@ -5,7 +8,7 @@ import base64
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import (
from vllm_omni.model_executor.models.nemotron_voicechat.duplex.input import (
NemotronVoiceChatPcmAppendBuffer,
)
@@ -10,12 +10,12 @@ import numpy as np
import pytest
from vllm.sampling_params import SamplingParams
from vllm_omni.experimental.fullduplex.engine.contracts import DuplexInputMode
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime import (
from vllm_omni.engine.duplex.contracts import DuplexInputMode
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.model_executor.models.nemotron_voicechat.duplex.runtime import (
NemotronVoiceChatDuplexRuntimeExtension,
)
from vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter import (
from vllm_omni.model_executor.models.nemotron_voicechat.duplex.serving_adapter import (
NemotronVoiceChatServingRuntimeAdapter,
_render_tool_response,
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import base64
from types import SimpleNamespace
@@ -8,11 +8,11 @@ import numpy as np
import pytest
import torch
from vllm_omni.experimental.fullduplex.personaplex.policy import (
from vllm_omni.model_executor.models.personaplex.duplex.policy import (
SILENCE_TOKENS,
SINE_TOKENS,
)
from vllm_omni.experimental.fullduplex.personaplex.stage0 import (
from vllm_omni.model_executor.models.personaplex.duplex.stage0 import (
PersonaPlexStage0DuplexRuntime,
)
from vllm_omni.model_executor.models.personaplex.personaplex_talker import (
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import base64
from pathlib import Path
@@ -14,25 +14,25 @@ from vllm_omni.config.stage_config import (
load_deploy_config,
merge_pipeline_deploy,
)
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.engine.duplex.runtime import (
DuplexInputMode,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.openai.runtime_adapter import (
from vllm_omni.entrypoints.duplex.runtime_adapter import (
ServingRuntimeConfigError,
)
from vllm_omni.experimental.fullduplex.personaplex.config import DEFAULT_PERSONA
from vllm_omni.experimental.fullduplex.personaplex.data_plane import (
from vllm_omni.model_executor.models.personaplex.duplex.config import DEFAULT_PERSONA
from vllm_omni.model_executor.models.personaplex.duplex.data_plane import (
PersonaPlexDataPlaneContext,
PersonaPlexDataPlaneSession,
)
from vllm_omni.experimental.fullduplex.personaplex.input import (
from vllm_omni.model_executor.models.personaplex.duplex.input import (
PersonaPlexPcmAppendBuffer,
)
from vllm_omni.experimental.fullduplex.personaplex.runtime_extension import (
from vllm_omni.model_executor.models.personaplex.duplex.runtime_extension import (
PersonaPlexDuplexRuntimeExtension,
)
from vllm_omni.experimental.fullduplex.personaplex.serving_adapter import (
from vllm_omni.model_executor.models.personaplex.duplex.serving_adapter import (
PersonaPlexServingRuntimeAdapter,
)
from vllm_omni.model_executor.models.personaplex.pipeline import (
@@ -184,11 +184,11 @@ def test_e2e_driver_rejects_unexpected_input_sha256(tmp_path: Path) -> None:
[
(
"duplex_runtime_extension",
"vllm_omni.experimental.fullduplex.personaplex.runtime_extension.PersonaPlexDuplexRuntimeExtension",
"vllm_omni.model_executor.models.personaplex.duplex.runtime_extension.PersonaPlexDuplexRuntimeExtension",
),
(
"duplex_serving_adapter",
"vllm_omni.experimental.fullduplex.personaplex.serving_adapter.PersonaPlexServingRuntimeAdapter",
"vllm_omni.model_executor.models.personaplex.duplex.serving_adapter.PersonaPlexServingRuntimeAdapter",
),
],
)
@@ -286,9 +286,14 @@ def test_personaplex_runtime_config_update_rejects_changed_voice() -> None:
assert unchanged["personaplex_voice_prompt"] == "NATF2.pt"
def test_personaplex_rejects_minicpmo_only_opt_out_flag() -> None:
def test_personaplex_rejects_native_duplex_opt_out_flag() -> None:
# PersonaPlex is always model-native; the per-session opt-in knob is
# server-owned there, under both the canonical name and its deprecated
# model-prefixed alias.
adapter = PersonaPlexServingRuntimeAdapter(lambda *_: None)
with pytest.raises(ServingRuntimeConfigError, match="native_duplex"):
adapter.validate_client_extra_body({"native_duplex": False})
with pytest.raises(ServingRuntimeConfigError, match="minicpmo45_native_duplex"):
adapter.validate_client_extra_body({"minicpmo45_native_duplex": False})
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
+32 -32
View File
@@ -18,7 +18,7 @@ def _minicpmo_duplex_policy_case(
state: SimpleNamespace,
payload: dict[str, object],
):
from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow
from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRow
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
MiniCPMO45OmniForConditionalGeneration,
)
@@ -45,7 +45,7 @@ def _minicpmo_duplex_policy_case(
def test_minicpmo_model_hook_owns_duplex_sampling_rows_and_force_listen():
from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingRow
from vllm_omni.model_executor.duplex_sampling import DuplexSamplingRow
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
MiniCPMO45OmniForConditionalGeneration,
)
@@ -329,7 +329,7 @@ def test_minicpmo_model_hook_ignores_serving_new_user_turn_marker():
def test_generic_ar_runner_builds_typed_duplex_sampling_rows():
from vllm_omni.experimental.fullduplex.model_executor import DuplexSamplingHelper
from vllm_omni.model_executor.duplex_sampling import DuplexSamplingHelper
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
runner = GPUARModelRunner.__new__(GPUARModelRunner)
@@ -532,7 +532,7 @@ def test_minicpmo_stage0_routes_duplex_metadata_per_batched_request():
def test_minicpmo_stage0_rejects_invalid_resolved_ref_audio():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -546,7 +546,7 @@ def test_minicpmo_stage0_rejects_invalid_resolved_ref_audio():
def test_minicpmo_stage0_special_token_ids_are_tokenizer_derived():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -583,7 +583,7 @@ def test_minicpmo_stage0_special_token_ids_are_tokenizer_derived():
def test_minicpmo_stage0_rejects_unknown_special_token_fallbacks():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -659,7 +659,7 @@ def _transformers_processor_stub(auto_processor):
def test_minicpmo_stage0_loads_processor_from_hf_id(monkeypatch):
import sys
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -701,7 +701,7 @@ def test_minicpmo_stage0_loads_processor_from_hf_id(monkeypatch):
def test_minicpmo_stage0_processor_load_preserves_original_exception(monkeypatch):
import sys
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -728,7 +728,7 @@ def test_minicpmo_stage0_processor_load_preserves_original_exception(monkeypatch
def test_minicpmo_stage0_processor_requires_tokenizer(monkeypatch):
import sys
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -751,7 +751,7 @@ def test_minicpmo_stage0_processor_requires_tokenizer(monkeypatch):
def test_minicpmo_stage0_loaded_processor_validates_special_tokens(monkeypatch):
import sys
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -793,7 +793,7 @@ def test_minicpmo_stage0_loaded_processor_validates_special_tokens(monkeypatch):
def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -855,7 +855,7 @@ def test_minicpmo_stage0_data_plane_prefill_matches_official_unit_format():
def _stage0_vision_runtime():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -912,7 +912,7 @@ def test_minicpmo_stage0_puts_every_frame_of_an_append_in_one_unit():
that closed a single unit left a frame over, failed the prefill, and still
advanced the session state.
"""
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
@@ -936,7 +936,7 @@ def test_minicpmo_stage0_wraps_hd_slices_after_the_source_image():
"""Official stacked pair is max_slice_nums=[2, 1]: source + slices, then audio."""
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
@@ -985,7 +985,7 @@ def test_minicpmo_stage0_reports_frames_that_close_no_unit_instead_of_dropping_t
Frames are not carried across appends, so one arriving early used to vanish
under a generic "audio not enough" buffering result.
"""
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
@@ -1010,7 +1010,7 @@ def test_minicpmo_stage0_reports_frames_that_close_no_unit_instead_of_dropping_t
def test_minicpmo_stage0_speech_append_sets_pending_context_once():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1118,7 +1118,7 @@ def test_minicpmo_stage0_speech_append_sets_pending_context_once():
def test_minicpmo_stage0_failed_append_does_not_set_pending_speech_context():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1173,7 +1173,7 @@ def test_minicpmo_stage0_failed_append_does_not_set_pending_speech_context():
def test_minicpmo_stage0_streaming_processor_is_isolated_per_session():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1220,7 +1220,7 @@ def test_minicpmo_stage0_streaming_processor_is_isolated_per_session():
def test_minicpmo_stage0_data_plane_next_append_reinjects_previous_listen():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1283,7 +1283,7 @@ def test_minicpmo_stage0_data_plane_next_append_reinjects_previous_listen():
def test_minicpmo_stage0_data_plane_turn_eos_closes_previous_unit():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1346,7 +1346,7 @@ def test_minicpmo_stage0_data_plane_turn_eos_closes_previous_unit():
def test_minicpmo_stage0_data_plane_model_owned_turn_boundary_preserves_audio_cache():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1411,7 +1411,7 @@ def test_minicpmo_stage0_data_plane_model_owned_turn_boundary_preserves_audio_ca
def test_minicpmo_stage0_data_plane_final_first_chunk_does_not_add_silence_unit():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1485,7 +1485,7 @@ def test_minicpmo_stage0_data_plane_final_first_chunk_does_not_add_silence_unit(
def test_minicpmo_stage0_final_does_not_promote_first_chunk_alignment_tail_to_unit():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1567,7 +1567,7 @@ def test_minicpmo_stage0_final_does_not_promote_first_chunk_alignment_tail_to_un
def test_minicpmo_stage0_runtime_uses_loaded_vllm_embed_tokens_when_get_input_embeddings_is_broken():
import torch
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
)
@@ -1603,7 +1603,7 @@ def test_minicpmo_stage0_runtime_uses_loaded_vllm_embed_tokens_when_get_input_em
def test_minicpmo_remote_config_patch_handles_nested_and_dict_configs():
from vllm_omni.experimental.fullduplex.minicpmo45.compat import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.compat import (
patch_minicpmo_remote_config,
)
@@ -1625,7 +1625,7 @@ def test_minicpmo_remote_config_patch_handles_nested_and_dict_configs():
def test_minicpmo_stage0_short_audio_buffers_without_context_mutation():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)
@@ -1701,7 +1701,7 @@ def test_minicpmo_stage0_native_sampler_penalizes_repeated_text_token():
def test_minicpmo_stage0_native_sampler_penalizes_text_from_prior_chunk():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
@@ -1761,7 +1761,7 @@ def test_minicpmo_stage0_native_sampler_penalizes_text_from_prior_chunk():
def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
@@ -1820,7 +1820,7 @@ def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty(
def test_minicpmo_stage0_records_bounded_model_policy_history():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
_MiniCPMO45Stage0SessionState,
)
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
@@ -2248,7 +2248,7 @@ def test_minicpmo_stage0_native_sampler_preserves_model_chunk_eos():
def test_minicpmo_stage0_native_sampler_keeps_hard_chunk_cap():
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
MiniCPMO45DuplexPolicy,
)
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
@@ -2360,7 +2360,7 @@ def test_minicpmo_stage0_native_sampler_cuts_before_request_length_cap():
def test_minicpmo_stage0_native_sampler_cuts_on_decoded_text_length():
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
MiniCPMO45DuplexPolicy,
)
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
@@ -2598,7 +2598,7 @@ def test_minicpmo_stage0_native_sampler_uses_runner_duplex_rows():
def test_minicpmo_stage0_session_context_includes_resolved_ref_audio():
from vllm_omni.experimental.fullduplex.minicpmo45.stage0 import (
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.stage0 import (
MiniCPMO45Stage0DuplexRuntime,
_MiniCPMO45Stage0SessionState,
)