[Core] Graduate MiniCPM-o 4.5 and PersonaPlex full-duplex serving out of experimental (#6196)
Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
179 files changed
+6462
-3832
No files matched your search
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.audio import convert_input_audio_with_rate
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_pcm16_input_conversion_is_pure_and_reports_target_rate():
|
||||
pcm16 = np.array([-32768, 0, 32767], dtype="<i2").tobytes()
|
||||
encoded = base64.b64encode(pcm16).decode("ascii")
|
||||
|
||||
converted, fmt, sample_rate = convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
|
||||
assert encoded == base64.b64encode(pcm16).decode("ascii")
|
||||
assert fmt == "pcm_f32le"
|
||||
assert sample_rate == 16_000
|
||||
samples = np.frombuffer(base64.b64decode(converted), dtype="<f4")
|
||||
assert np.allclose(samples, [-1.0, 0.0, 32767 / 32768])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate_hz", [8_000, 192_000])
|
||||
def test_pcm16_input_conversion_accepts_supported_sample_rate_boundaries(sample_rate_hz: int):
|
||||
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
|
||||
|
||||
_, fmt, converted_rate = convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=sample_rate_hz,
|
||||
)
|
||||
|
||||
assert fmt == "pcm_f32le"
|
||||
assert converted_rate == 16_000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate_hz", [7_999, 192_001, 10**1000])
|
||||
def test_pcm16_input_conversion_rejects_sample_rates_outside_supported_range(sample_rate_hz: int):
|
||||
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
|
||||
|
||||
with pytest.raises(ValueError, match="sample_rate_hz"):
|
||||
convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=sample_rate_hz,
|
||||
)
|
||||
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.runtime_adapter import (
|
||||
validate_serving_runtime_adapter,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_name",
|
||||
[
|
||||
"vllm_omni.entrypoints.duplex.session_runner",
|
||||
"vllm_omni.entrypoints.duplex.runtime_bridge",
|
||||
"vllm_omni.entrypoints.duplex.serving",
|
||||
],
|
||||
)
|
||||
def test_generic_openai_runtime_import_does_not_load_model_adapters(module_name: str) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = f"""
|
||||
import sys
|
||||
|
||||
import {module_name}
|
||||
|
||||
model_duplex_prefixes = (
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
|
||||
"vllm_omni.model_executor.models.nemotron_voicechat.duplex",
|
||||
"vllm_omni.model_executor.models.personaplex.duplex",
|
||||
)
|
||||
loaded = sorted(
|
||||
name
|
||||
for name in sys.modules
|
||||
if any(
|
||||
name == prefix or name.startswith(prefix + ".")
|
||||
for prefix in model_duplex_prefixes
|
||||
)
|
||||
)
|
||||
if loaded:
|
||||
raise SystemExit("generic OpenAI runtime loaded model duplex modules: " + ", ".join(loaded))
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_generic_handler_requires_explicit_serving_runtime_adapter() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = """
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
|
||||
chat_service = SimpleNamespace(engine_client=SimpleNamespace())
|
||||
try:
|
||||
OmniDuplexSessionHandler(chat_service=chat_service)
|
||||
except ValueError as exc:
|
||||
if "serving runtime adapter" not in str(exc).lower():
|
||||
raise
|
||||
else:
|
||||
raise SystemExit("generic handler silently selected a model serving adapter")
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def _valid_runtime_adapter() -> SimpleNamespace:
|
||||
data_plane = SimpleNamespace(
|
||||
begin_request=lambda request_id: None,
|
||||
is_terminal=lambda request_id: False,
|
||||
mark_terminal=lambda request_id: None,
|
||||
close_stream=lambda request_id: None,
|
||||
close_session=lambda session_id, **kwargs: None,
|
||||
project=lambda result, **kwargs: (),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
adapter_id="test-adapter",
|
||||
session_states={},
|
||||
data_plane=data_plane,
|
||||
clean_response_done_prefix="clean",
|
||||
interrupted_tts_prefix="interrupted",
|
||||
private_runtime_config_keys=frozenset({"private_key"}),
|
||||
create_session_state=lambda: object(),
|
||||
session_state=lambda session_id: object(),
|
||||
remove_session_state=lambda session_id: None,
|
||||
is_enabled=lambda config: True,
|
||||
capabilities=lambda **kwargs: object(),
|
||||
validate_client_extra_body=lambda extra_body: None,
|
||||
prepare_runtime_config=lambda config, **kwargs: {},
|
||||
runtime_config_for_update=lambda config, current: {},
|
||||
data_plane_context=lambda **kwargs: object(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attribute", "invalid_value"),
|
||||
[
|
||||
("session_states", None),
|
||||
("clean_response_done_prefix", None),
|
||||
("interrupted_tts_prefix", None),
|
||||
("private_runtime_config_keys", {"private_key"}),
|
||||
],
|
||||
)
|
||||
def test_runtime_adapter_validator_rejects_invalid_protocol_attributes(
|
||||
attribute: str,
|
||||
invalid_value: object,
|
||||
) -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
setattr(adapter, attribute, invalid_value)
|
||||
|
||||
with pytest.raises(TypeError, match=attribute):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
|
||||
|
||||
def test_runtime_adapter_validator_rejects_incomplete_data_plane() -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
adapter.data_plane.close_session = None
|
||||
|
||||
with pytest.raises(TypeError, match="data_plane.*close_session"):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Regression tests for duplex ``runtime_control`` payload redaction.
|
||||
|
||||
The redacted result is emitted on the WebSocket (for example
|
||||
``session.created.runtime_control``), so it must always survive
|
||||
``json.dumps``. Control results carry typed values straight from the engine --
|
||||
``fence`` and ``accepted_fence`` are ``DuplexFence`` instances added by
|
||||
``DuplexControlClient.execute`` -- which previously reached
|
||||
``WebSocket.send_json`` unconverted and killed the session at handshake.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.entrypoints.duplex.runtime_bridge import NativeRuntimeBridgeMixin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_redact = NativeRuntimeBridgeMixin._redact_runtime_control_result
|
||||
|
||||
|
||||
def _control_result() -> dict[str, object]:
|
||||
"""The shape ``DuplexControlClient.execute`` returns for an open."""
|
||||
return {
|
||||
"ok": True,
|
||||
"operation": "open",
|
||||
"session_id": "sess-1",
|
||||
"fence": DuplexFence("sess-1", epoch=0, turn_id=0, response_seq=0, incarnation=0),
|
||||
"accepted_fence": DuplexFence("sess-1", epoch=1, turn_id=2, response_seq=3, incarnation=4),
|
||||
"stage_results": [{"stage_id": -1, "replica_id": -1, "result": {"supported": True}}],
|
||||
}
|
||||
|
||||
|
||||
def test_redacted_control_result_is_json_serializable() -> None:
|
||||
"""The whole payload must serialize the way send_json does."""
|
||||
redacted = _redact(_control_result())
|
||||
# Must not raise.
|
||||
json.dumps({"type": "session.created", "runtime_control": redacted})
|
||||
|
||||
|
||||
def test_fences_are_converted_to_plain_dicts() -> None:
|
||||
redacted = _redact(_control_result())
|
||||
assert redacted["fence"] == {
|
||||
"session_id": "sess-1",
|
||||
"epoch": 0,
|
||||
"turn_id": 0,
|
||||
"response_seq": 0,
|
||||
"incarnation": 0,
|
||||
}
|
||||
# Field values are preserved, not just the type.
|
||||
assert redacted["accepted_fence"]["epoch"] == 1
|
||||
assert redacted["accepted_fence"]["turn_id"] == 2
|
||||
assert redacted["accepted_fence"]["response_seq"] == 3
|
||||
assert redacted["accepted_fence"]["incarnation"] == 4
|
||||
|
||||
|
||||
def test_nested_dataclasses_are_converted() -> None:
|
||||
"""Fences can appear inside stage results, not only at the top level."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"stage_results": [
|
||||
{"stage_id": 0, "result": {"fence": DuplexFence("sess-2", epoch=7)}},
|
||||
]
|
||||
}
|
||||
)
|
||||
json.dumps(redacted)
|
||||
assert redacted["stage_results"][0]["result"]["fence"]["epoch"] == 7
|
||||
|
||||
|
||||
def test_enums_are_reduced_to_values() -> None:
|
||||
class _Mode(str, Enum):
|
||||
APPEND = "append_audio_chunk"
|
||||
|
||||
redacted = _redact({"mode": _Mode.APPEND})
|
||||
json.dumps(redacted)
|
||||
assert redacted["mode"] == "append_audio_chunk"
|
||||
|
||||
|
||||
def test_redaction_still_drops_heavy_payload_keys() -> None:
|
||||
"""The pre-existing contract must be preserved."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"ok": True,
|
||||
"stage_handoff": object(),
|
||||
"tts_handoff": object(),
|
||||
"omni_payload": object(),
|
||||
"tts_hidden_states": object(),
|
||||
"tts_token_ids": object(),
|
||||
"traceback": "long trace",
|
||||
}
|
||||
)
|
||||
assert redacted == {"ok": True}
|
||||
|
||||
|
||||
def test_plain_values_pass_through_unchanged() -> None:
|
||||
payload = {"ok": True, "count": 3, "name": "x", "missing": None, "items": [1, "two"]}
|
||||
assert _redact(payload) == payload
|
||||
|
||||
|
||||
def test_dataclass_types_are_not_treated_as_instances() -> None:
|
||||
"""A class object is not a dataclass instance and must not be asdict'd."""
|
||||
|
||||
@dataclass
|
||||
class _Shape:
|
||||
x: int = 1
|
||||
|
||||
assert _redact(_Shape) is _Shape
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.websocket import (
|
||||
DuplexWebSocketActor,
|
||||
normalize_duplex_input_event,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
def __init__(self) -> None:
|
||||
self.sent: list[dict[str, object]] = []
|
||||
|
||||
async def send_json(self, payload: dict[str, object]) -> None:
|
||||
self.sent.append(dict(payload))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event", "expected"),
|
||||
[
|
||||
({"type": "signal_turn", "event": "barge_in"}, {"type": "turn.signal", "event": "barge_in"}),
|
||||
({"type": "close_session"}, {"type": "session.close"}),
|
||||
({"type": "audio.playback_ack", "played_ms": 1}, {"type": "playback.ack", "played_ms": 1}),
|
||||
({"type": "input_text.append", "text": "a"}, {"type": "input.text.append", "text": "a"}),
|
||||
({"type": "push_text", "text": "b"}, {"type": "input.text.append", "text": "b"}),
|
||||
(
|
||||
{"type": "input.audio.append", "audio": "wav"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "wav"},
|
||||
),
|
||||
(
|
||||
{"type": "push_chunk", "audio": "wav", "format": "pcm_f32le"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "pcm_f32le"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_input_aliases_normalize_once_at_mailbox_boundary(event, expected):
|
||||
assert normalize_duplex_input_event(event) == expected
|
||||
|
||||
|
||||
def test_actor_uses_one_fifo_mailbox_for_inbound_events():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
|
||||
assert isinstance(actor.mailbox, asyncio.Queue)
|
||||
assert not isinstance(actor.mailbox, asyncio.PriorityQueue)
|
||||
assert not hasattr(actor, "control_queue")
|
||||
assert not hasattr(actor, "input_queue")
|
||||
assert not hasattr(actor, "event_queue")
|
||||
assert not hasattr(actor, "lifecycle_state")
|
||||
assert not hasattr(actor, "last_response_id")
|
||||
assert not hasattr(actor, "overlap_speech_ms")
|
||||
assert not hasattr(actor, "transition")
|
||||
assert not hasattr(actor, "output_generation_in_flight")
|
||||
assert not hasattr(actor, "session")
|
||||
assert not hasattr(actor, "runtime_opened")
|
||||
assert not hasattr(actor, "runtime_closed")
|
||||
assert not hasattr(actor, "drain_input_queue")
|
||||
assert not hasattr(actor, "control_events_seen")
|
||||
assert not hasattr(actor, "input_events_seen")
|
||||
assert not hasattr(actor, "cancel_count")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_event_does_not_overtake_earlier_audio_input():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": "response.cancel"})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": "response.cancel"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("event_type", ["input.cancel", "session.close", "close_session"])
|
||||
async def test_terminal_control_preserves_wire_order(event_type: str):
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": event_type})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": event_type}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mailbox_delivers_every_enqueued_event_exactly_once():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
events = [
|
||||
{"type": "input_audio_buffer.append", "audio": str(index)}
|
||||
if index % 2
|
||||
else {"type": "response.created", "response_id": f"resp-{index}"}
|
||||
for index in range(20)
|
||||
]
|
||||
|
||||
await asyncio.gather(*(actor.enqueue_event(dict(event)) for event in events))
|
||||
received = [await actor.next_event() for _ in events]
|
||||
|
||||
assert sorted(received, key=repr) == sorted(events, key=repr)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_is_single_owner_of_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "one"})
|
||||
await actor.send_json({"type": "two"})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "one"}, {"type": "two"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_fence_payload_is_dropped_before_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 2)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "response.audio.delta", "epoch": 1})
|
||||
await actor.send_json({"type": "response.audio.delta", "epoch": 2})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "response.audio.delta", "epoch": 2}]
|
||||
assert actor.stale_output_dropped == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_does_not_revoke_accepted_terminal_after_close_starts():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 1)
|
||||
await actor.send_json({"type": "response.done", "epoch": 1, "response_id": "resp-1"})
|
||||
actor.closing = True
|
||||
await actor.close_writer()
|
||||
|
||||
await actor.writer_loop()
|
||||
|
||||
assert websocket.sent == [{"type": "response.done", "epoch": 1, "response_id": "resp-1"}]
|
||||
assert actor.stale_output_dropped == 0
|
||||
@@ -1,10 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.openai.protocol import (
|
||||
DuplexCapabilities,
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexOverlapPolicy,
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
@@ -14,10 +16,13 @@ from vllm_omni.experimental.fullduplex.openai.protocol import (
|
||||
DuplexTurnState,
|
||||
ResponseCreateOptions,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.openai.realtime_session import (
|
||||
from vllm_omni.entrypoints.duplex.realtime_session import (
|
||||
NativeRealtimeSessionProtocol,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.openai.realtime_state import RealtimeStateOwner
|
||||
from vllm_omni.entrypoints.duplex.realtime_state import RealtimeStateOwner
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
|
||||
minicpmo45_native_capabilities,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -116,12 +121,28 @@ def test_realtime_model_name_does_not_implicitly_enable_native_duplex():
|
||||
|
||||
event = protocol._session_create_from_realtime({"model": "openbmb/MiniCPM-o-4_5"})
|
||||
|
||||
assert "native_duplex" not in event["session"]["extra_body"]
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
def test_realtime_explicit_native_duplex_flag_is_preserved():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_body": {"native_duplex": True},
|
||||
}
|
||||
)
|
||||
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
|
||||
|
||||
def test_realtime_legacy_native_duplex_alias_is_normalized():
|
||||
# The deprecated model-prefixed spelling still opts in, but everything
|
||||
# downstream (and every session echo) sees only the canonical key.
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
@@ -129,10 +150,25 @@ def test_realtime_explicit_native_duplex_flag_is_preserved():
|
||||
}
|
||||
)
|
||||
|
||||
assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostart():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
|
||||
|
||||
assert event["type"] == "session.create"
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
|
||||
|
||||
def test_realtime_legacy_query_native_duplex_alias_seeds_canonical_key():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
@@ -143,7 +179,8 @@ def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostar
|
||||
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
|
||||
|
||||
assert event["type"] == "session.create"
|
||||
assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -151,7 +188,7 @@ async def test_realtime_resume_heartbeat_and_event_ack_translate_without_session
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"minicpmo45_native_duplex": "1",
|
||||
"native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -217,7 +254,7 @@ async def test_realtime_resume_query_suppresses_model_autostart_on_same_url():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"minicpmo45_native_duplex": "1",
|
||||
"native_duplex": "1",
|
||||
"resume": "1",
|
||||
}
|
||||
)
|
||||
@@ -293,6 +330,39 @@ def test_realtime_projects_resume_lifecycle_events_without_duplex_prefix():
|
||||
}
|
||||
|
||||
|
||||
def test_realtime_projects_listen_response_identity():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
protocol.encode_outbound_event({"type": "session.created", "session": {"id": "sid-listen"}})
|
||||
|
||||
bound = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "response.listen",
|
||||
"session_id": "sid-listen",
|
||||
"epoch": 0,
|
||||
"reason": "model_listen",
|
||||
"model_listen": True,
|
||||
"response_id": "resp_1",
|
||||
}
|
||||
)[0]
|
||||
# A listen that terminates a precreated response keeps the response
|
||||
# identity on the Realtime dialect; clients demultiplex on response_id.
|
||||
assert bound["type"] == "response.listen"
|
||||
assert bound["response_id"] == "resp_1"
|
||||
assert bound["response"]["id"] == "resp_1"
|
||||
assert bound["response"]["status"] == "listening"
|
||||
|
||||
anonymous = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "response.listen",
|
||||
"session_id": "sid-listen",
|
||||
"epoch": 0,
|
||||
"reason": "silence_or_noise",
|
||||
}
|
||||
)[0]
|
||||
assert "response_id" not in anonymous
|
||||
assert "id" not in anonymous["response"]
|
||||
|
||||
|
||||
def test_duplex_session_registry_advances_incarnation_when_id_is_reused():
|
||||
registry = DuplexSessionRegistry()
|
||||
|
||||
@@ -436,7 +506,7 @@ def test_duplex_capabilities_do_not_claim_core_kv_or_input_append():
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease():
|
||||
caps = DuplexCapabilities.minicpmo45_native(max_sessions=2).as_dict()
|
||||
caps = minicpmo45_native_capabilities(max_sessions=2).as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "model_native_duplex"
|
||||
assert caps["supports_input_append"] is True
|
||||
@@ -463,7 +533,7 @@ def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease():
|
||||
|
||||
|
||||
def test_duplex_overlap_policy_auto_falls_back_to_listen_only():
|
||||
from vllm_omni.experimental.fullduplex.openai.protocol import (
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexOverlapPolicy,
|
||||
DuplexSessionConfig,
|
||||
)
|
||||
@@ -472,7 +542,7 @@ def test_duplex_overlap_policy_auto_falls_back_to_listen_only():
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment():
|
||||
caps = DuplexCapabilities.minicpmo45_native(max_sessions=1).as_dict()
|
||||
caps = minicpmo45_native_capabilities(max_sessions=1).as_dict()
|
||||
|
||||
assert caps["supports_multi_session"] is False
|
||||
assert caps["supports_multi_session_same_replica"] is False
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -9,7 +9,7 @@ import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.openai.session_attachment import (
|
||||
from vllm_omni.entrypoints.duplex.session_attachment import (
|
||||
DuplexEventJournal,
|
||||
DuplexJournalGapError,
|
||||
DuplexJournalOverflowError,
|
||||
@@ -43,7 +43,7 @@ def test_resume_credential_uses_256_bit_token_digest_and_redacted_repr(monkeypat
|
||||
credential = DuplexResumeCredential.from_token(token)
|
||||
calls = []
|
||||
|
||||
import vllm_omni.experimental.fullduplex.openai.session_attachment as attachment_module
|
||||
import vllm_omni.entrypoints.duplex.session_attachment as attachment_module
|
||||
|
||||
original_compare = attachment_module.hmac.compare_digest
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
@@ -7,13 +10,37 @@ import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_REPO_ROOT = Path(__file__).parents[3]
|
||||
_CLIENTS_PACKAGE = "vllm_omni.clients"
|
||||
|
||||
def test_api_server_does_not_import_experimental_duplex_at_module_load() -> None:
|
||||
api_server = Path(__file__).parents[3] / "vllm_omni/entrypoints/openai/api_server.py"
|
||||
module = ast.parse(api_server.read_text())
|
||||
|
||||
top_level_imports = {
|
||||
node.module for node in module.body if isinstance(node, ast.ImportFrom) and isinstance(node.module, str)
|
||||
}
|
||||
def _imports_clients_package(path: Path) -> bool:
|
||||
"""True if the module imports the client-side package at module level.
|
||||
|
||||
assert not any(module.startswith("vllm_omni.experimental.fullduplex") for module in top_level_imports)
|
||||
``vllm_omni.clients`` is client-side by construction: the serving stack
|
||||
(imported eagerly by the API server) must never depend on it.
|
||||
"""
|
||||
module = ast.parse(path.read_text(encoding="utf-8"))
|
||||
for node in module.body:
|
||||
if isinstance(node, ast.Import):
|
||||
if any(
|
||||
alias.name == _CLIENTS_PACKAGE or alias.name.startswith(_CLIENTS_PACKAGE + ".") for alias in node.names
|
||||
):
|
||||
return True
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module and (node.module == _CLIENTS_PACKAGE or node.module.startswith(_CLIENTS_PACKAGE + ".")):
|
||||
return True
|
||||
if node.module == "vllm_omni" and any(alias.name == "clients" for alias in node.names):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def test_api_server_does_not_import_client_package() -> None:
|
||||
api_server = _REPO_ROOT / "vllm_omni/entrypoints/openai/api_server.py"
|
||||
assert not _imports_clients_package(api_server)
|
||||
|
||||
|
||||
def test_duplex_serving_stack_does_not_import_client_package() -> None:
|
||||
duplex_pkg = _REPO_ROOT / "vllm_omni/entrypoints/duplex"
|
||||
offenders = [path.name for path in sorted(duplex_pkg.glob("*.py")) if _imports_clients_package(path)]
|
||||
assert not offenders, f"duplex serving modules import the client package: {offenders}"
|
||||
@@ -14,46 +14,59 @@ import numpy as np
|
||||
import pytest
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
# Import the OpenAI entrypoints package at collection time, before pytest
|
||||
# attaches its caplog handlers to the root logger. vllm's api_server
|
||||
# transitively imports model_hosting_container_standards, whose import-time
|
||||
# configure_root_logger() raises the level of every handler then attached to
|
||||
# the root logger to ERROR. The serving runtime under test reaches this chain
|
||||
# lazily (runtime_bridge -> entrypoints.openai.protocol.audio), so without
|
||||
# this eager import the side effect fires mid-test and silences caplog for
|
||||
# the remainder of the session. Before the capability predicate moved to
|
||||
# vllm_omni.entrypoints.duplex.capability, this file imported the package at
|
||||
# module scope implicitly.
|
||||
import vllm_omni.entrypoints.openai # noqa: F401
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_control_client import DuplexControlRequestError
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
|
||||
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence, DuplexSessionLifecycleMessage
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45 import (
|
||||
MiniCPMO45NativeDuplexServingAdapter,
|
||||
MiniCPMO45PcmAppendBuffer,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.data_plane import (
|
||||
MiniCPMO45DataPlaneContext,
|
||||
MiniCPMO45DataPlaneSession,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.runtime import (
|
||||
MiniCPMO45DuplexRuntimeExtension,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.session import (
|
||||
MiniCPMO45ServingSessionState,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.openai import vad as realtime_vad
|
||||
from vllm_omni.experimental.fullduplex.openai.protocol import (
|
||||
DuplexCapabilities,
|
||||
from vllm_omni.engine.duplex.control_client import DuplexControlRequestError
|
||||
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence, DuplexSessionLifecycleMessage
|
||||
from vllm_omni.engine.duplex.runtime import duplex_resource_request_id
|
||||
from vllm_omni.entrypoints.duplex import vad as realtime_vad
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexOverlapPolicy,
|
||||
DuplexPlaybackCommitPolicy,
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
ResponseCreateOptions,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.openai.realtime_session import NativeRealtimeSessionProtocol
|
||||
from vllm_omni.experimental.fullduplex.openai.runtime_adapter import ServingRuntimeConfigError
|
||||
from vllm_omni.experimental.fullduplex.openai.serving import (
|
||||
from vllm_omni.entrypoints.duplex.realtime_session import NativeRealtimeSessionProtocol
|
||||
from vllm_omni.entrypoints.duplex.runtime_adapter import ServingRuntimeConfigError
|
||||
from vllm_omni.entrypoints.duplex.serving import (
|
||||
OmniDuplexSessionHandler,
|
||||
should_enable_duplex_endpoint,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.openai.websocket import DuplexWebSocketActor
|
||||
from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision
|
||||
from vllm_omni.experimental.fullduplex.personaplex.serving_adapter import (
|
||||
from vllm_omni.entrypoints.duplex.websocket import DuplexWebSocketActor
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex import (
|
||||
MiniCPMO45NativeDuplexServingAdapter,
|
||||
MiniCPMO45PcmAppendBuffer,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
|
||||
minicpmo45_native_capabilities,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import (
|
||||
MiniCPMO45DataPlaneContext,
|
||||
MiniCPMO45DataPlaneSession,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import (
|
||||
MiniCPMO45DuplexRuntimeExtension,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.session import (
|
||||
MiniCPMO45ServingSessionState,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.duplex.serving_adapter import (
|
||||
PersonaPlexServingRuntimeAdapter,
|
||||
)
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.outputs.duplex import attach_duplex_output_decision
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -62,12 +75,12 @@ def test_native_input_append_supports_explicit_session_opt_out():
|
||||
session = DuplexSession(
|
||||
"sid-explicit-tts",
|
||||
DuplexSessionConfig(model="test-model"),
|
||||
capabilities=DuplexCapabilities.minicpmo45_native(),
|
||||
capabilities=minicpmo45_native_capabilities(),
|
||||
)
|
||||
|
||||
assert OmniDuplexSessionHandler._uses_native_input_append(session) is True
|
||||
|
||||
session.config.extra_body["minicpmo45_native_duplex"] = False
|
||||
session.config.extra_body["native_duplex"] = False
|
||||
assert OmniDuplexSessionHandler._uses_native_input_append(session) is False
|
||||
|
||||
|
||||
@@ -293,7 +306,7 @@ class FakeEngineClient:
|
||||
|
||||
class FakeChatService:
|
||||
duplex_serving_adapter_path = (
|
||||
"vllm_omni.experimental.fullduplex.minicpmo45.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
|
||||
)
|
||||
|
||||
def __init__(self, engine_client: FakeEngineClient) -> None:
|
||||
@@ -844,7 +857,7 @@ def test_native_response_options_ignore_private_runtime_config():
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-private-config",
|
||||
config=DuplexSessionConfig(),
|
||||
capabilities=DuplexCapabilities.minicpmo45_native(),
|
||||
capabilities=minicpmo45_native_capabilities(),
|
||||
)
|
||||
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
|
||||
@@ -878,7 +891,7 @@ def test_native_response_create_rejects_unwired_generation_options(payload):
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-unwired-options",
|
||||
config=DuplexSessionConfig(),
|
||||
capabilities=DuplexCapabilities.minicpmo45_native(),
|
||||
capabilities=minicpmo45_native_capabilities(),
|
||||
)
|
||||
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(FakeEngineClient()))
|
||||
@@ -1244,7 +1257,7 @@ async def test_minicpmo_native_session_update_rejects_native_duplex_mode_flip():
|
||||
{
|
||||
"type": "turn.signal",
|
||||
"event": "session.update",
|
||||
"payload": {"extra_body": {"minicpmo45_native_duplex": False}},
|
||||
"payload": {"extra_body": {"native_duplex": False}},
|
||||
}
|
||||
)
|
||||
ws.put(
|
||||
@@ -1581,7 +1594,7 @@ def _native_session_create(
|
||||
event["session"]["model"] = "openbmb/MiniCPM-o-4_5"
|
||||
event["session"]["modalities"] = list(modalities or ["text"])
|
||||
event["session"]["instructions"] = "You are a concise assistant."
|
||||
event["session"]["extra_body"] = {"minicpmo45_native_duplex": True}
|
||||
event["session"]["extra_body"] = {"native_duplex": True}
|
||||
return event
|
||||
|
||||
|
||||
@@ -1598,7 +1611,7 @@ def _native_realtime_session_update(
|
||||
"modalities": list(modalities or ["text"]),
|
||||
"instructions": "You are a concise assistant.",
|
||||
"idle_timeout_s": 1,
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
"extra_body": {"native_duplex": True},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1674,7 +1687,7 @@ def _install_direct_silence_scheduler(
|
||||
handler: OmniDuplexSessionHandler,
|
||||
session: DuplexSession,
|
||||
) -> None:
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
|
||||
async def _schedule(payload: object, **kwargs: Any) -> bool:
|
||||
if handler._native_silence_continuation_is_stale(
|
||||
@@ -1819,7 +1832,7 @@ def test_legacy_auto_overlap_policy_falls_back_to_listen_only():
|
||||
overlap_policy=DuplexSessionConfig._normalize_overlap_policy("auto"),
|
||||
),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
|
||||
decision = handler._overlap_decision(
|
||||
session,
|
||||
@@ -2066,11 +2079,11 @@ def test_auto_response_post_response_silence_remains_model_owned_input():
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_response_terminal_advances_open_realtime_input_without_commit():
|
||||
handler, session = _auto_response_context("sid-auto-realtime-open-input", playback_active=True)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
response_id = session.active_response_id
|
||||
assert response_id is not None
|
||||
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
native.input_since_commit = True
|
||||
native.speech_since_commit = True
|
||||
session.accumulate_overlap_speech(800)
|
||||
@@ -2113,11 +2126,11 @@ async def test_auto_response_terminal_allows_next_model_unit_while_playback_drai
|
||||
"sid-auto-realtime-open-before-overlap",
|
||||
playback_active=True,
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
response_id = session.active_response_id
|
||||
assert response_id is not None
|
||||
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
native.input_since_commit = True
|
||||
native.speech_since_commit = True
|
||||
session.end_response(commit_text=False, preserve_request=True)
|
||||
@@ -2147,11 +2160,11 @@ async def test_auto_response_terminal_allows_next_model_unit_while_playback_drai
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_committed_overlap_promotes_after_response_done():
|
||||
handler, session = _auto_response_context("sid-auto-realtime-committed-input", playback_active=True)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
response_id = session.active_response_id
|
||||
assert response_id is not None
|
||||
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
native.committed_audio_payload = _native_audio_payload()
|
||||
native.deferred_response_create = True
|
||||
session.accumulate_overlap_speech(1000)
|
||||
@@ -3061,7 +3074,7 @@ async def test_native_append_propagates_current_turn_fence_to_engine():
|
||||
session_id="sid-fenced-append",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.turn_id = 2
|
||||
session.begin_response(turn_id=2)
|
||||
ws = TimedWebSocket()
|
||||
@@ -3091,7 +3104,7 @@ async def test_minicpmo_auto_response_tts_segment_boundary_appends_silence_unit(
|
||||
session_id="sid-segment-boundary",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.begin_response(turn_id=0)
|
||||
session.turn_id = 1
|
||||
session.bind_request(request_id)
|
||||
@@ -3140,7 +3153,7 @@ async def test_minicpmo_auto_response_pre_response_tts_boundary_continues_model_
|
||||
session_id="sid-pre-response-boundary",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.bind_request(request_id)
|
||||
_install_direct_silence_scheduler(handler, session)
|
||||
ws = TimedWebSocket()
|
||||
@@ -3187,11 +3200,11 @@ async def test_minicpmo_pre_response_continuation_drops_after_model_turn_ends():
|
||||
session_id="sid-stale-pre-response-boundary",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.bind_request(request_id)
|
||||
model_turn_id = session.turn_id
|
||||
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
|
||||
async def _stale_before_append(payload: object, **kwargs: Any) -> bool:
|
||||
session.complete_model_turn(model_turn_id)
|
||||
@@ -3237,7 +3250,7 @@ async def test_minicpmo_auto_response_continuation_stops_at_large_safety_boundar
|
||||
session_id="sid-long-response",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.begin_response(turn_id=0)
|
||||
session.turn_id = 1
|
||||
session.bind_request(request_id)
|
||||
@@ -3283,10 +3296,10 @@ async def test_minicpmo_auto_response_boundary_listen_closes_response():
|
||||
session_id="sid-boundary-listen",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
response_id = session.begin_response(turn_id=0)
|
||||
session.bind_request(request_id)
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
native.continuation_owner_id = f"response:{response_id}"
|
||||
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
|
||||
_install_direct_silence_scheduler(handler, session)
|
||||
@@ -3313,6 +3326,11 @@ async def test_minicpmo_auto_response_boundary_listen_closes_response():
|
||||
assert emitted is True
|
||||
assert engine.appended == []
|
||||
assert ws.sent_types() == ["response.listen", "response.done"]
|
||||
# The listen terminates the precreated response: clients demultiplex the
|
||||
# decision by its id, so the payload must carry the response identity.
|
||||
listen_payload, done_payload = ws.sent[-2], ws.sent[-1]
|
||||
assert listen_payload["response_id"] == response_id
|
||||
assert done_payload["response_id"] == response_id
|
||||
assert session.active_response_id is None
|
||||
assert session.active_request_id == request_id
|
||||
assert session.turn_id == 1
|
||||
@@ -3327,10 +3345,10 @@ async def test_minicpmo_auto_response_boundary_allows_next_model_turn_response()
|
||||
session_id="sid-boundary-next-turn",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.begin_response(turn_id=0)
|
||||
session.bind_request(request_id)
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
native.continuation_owner_id = f"response:{session.active_response_id}"
|
||||
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
|
||||
ws = TimedWebSocket()
|
||||
@@ -3382,10 +3400,10 @@ async def test_minicpmo_auto_response_boundary_does_not_close_new_epoch_response
|
||||
session_id="sid-boundary-race",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
old_response_id = session.begin_response(turn_id=0)
|
||||
session.bind_request(request_id)
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
native.continuation_owner_id = f"response:{old_response_id}"
|
||||
native.continuation_units = handler._NATIVE_AUTO_RESPONSE_MAX_CONTINUATION_UNITS
|
||||
sent: list[dict[str, Any]] = []
|
||||
@@ -3418,7 +3436,7 @@ async def test_minicpmo_auto_response_pre_speak_listen_continues_same_response()
|
||||
session_id="sid-pre-speak-listen",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
response_id = session.begin_response(turn_id=0)
|
||||
session.turn_id = 1
|
||||
session.bind_request(request_id)
|
||||
@@ -3458,7 +3476,7 @@ async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_req
|
||||
session_id="sid-listen-before-response",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.bind_request(request_id)
|
||||
ws = TimedWebSocket()
|
||||
|
||||
@@ -3482,7 +3500,7 @@ async def test_minicpmo_auto_response_listen_before_response_keeps_resumable_req
|
||||
assert close_reason is None
|
||||
assert emitted is True
|
||||
assert session.active_request_id == request_id
|
||||
assert not handler._minicpmo_data_plane.is_terminal(request_id)
|
||||
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
|
||||
assert ws.sent_types().count("response.listen") == 1
|
||||
|
||||
|
||||
@@ -3666,7 +3684,7 @@ async def test_minicpmo_auto_response_post_speak_listen_continues_same_response(
|
||||
session_id="sid-post-speak-listen",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
response_id = session.begin_response(turn_id=0)
|
||||
session.turn_id = 1
|
||||
session.bind_request(request_id)
|
||||
@@ -3707,7 +3725,7 @@ async def test_minicpmo_auto_response_turn_end_preserves_resumable_request():
|
||||
session_id="sid-turn-end-preserve-request",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.begin_response(turn_id=0)
|
||||
session.bind_request(request_id)
|
||||
ws = TimedWebSocket()
|
||||
@@ -3746,7 +3764,7 @@ async def test_minicpmo_auto_response_empty_turn_end_emits_model_listen():
|
||||
session_id="sid-empty-turn-end",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.bind_request(request_id)
|
||||
ws = TimedWebSocket()
|
||||
|
||||
@@ -3789,7 +3807,7 @@ async def test_minicpmo_auto_response_drops_late_audio_from_completed_model_turn
|
||||
session_id="sid-late-completed-turn",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.bind_request(request_id)
|
||||
session.complete_model_turn(0)
|
||||
ws = TimedWebSocket()
|
||||
@@ -4015,7 +4033,7 @@ async def test_duplex_auto_response_empty_terminal_does_not_create_empty_respons
|
||||
)
|
||||
|
||||
native_results = list(
|
||||
handler._minicpmo_data_plane.project(
|
||||
handler._serving_runtime_adapter.data_plane.project(
|
||||
{"data_plane_outputs": [output]},
|
||||
context=_data_plane_context(session),
|
||||
)
|
||||
@@ -4030,7 +4048,7 @@ async def test_duplex_auto_response_empty_terminal_does_not_create_empty_respons
|
||||
assert "response.created" not in [m.get("type") for m in sent]
|
||||
assert "response.done" not in [m.get("type") for m in sent]
|
||||
assert session.turn_id == 1
|
||||
assert not handler._minicpmo_data_plane.is_terminal(request_id)
|
||||
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -4040,11 +4058,11 @@ async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_pl
|
||||
engine = FakeEngineClient()
|
||||
chat_service = FakeChatService(engine)
|
||||
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
|
||||
handler._minicpmo_data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32))
|
||||
handler._minicpmo_data_plane.segment_text_delta(request_id, "hello")
|
||||
handler._minicpmo_data_plane.mark_terminal(request_id)
|
||||
handler._serving_runtime_adapter.data_plane.slice_cumulative_audio(request_id, np.zeros(24000, dtype=np.float32))
|
||||
handler._serving_runtime_adapter.data_plane.segment_text_delta(request_id, "hello")
|
||||
handler._serving_runtime_adapter.data_plane.mark_terminal(request_id)
|
||||
native = MiniCPMO45ServingSessionState()
|
||||
handler._minicpmo_sessions[session_id] = native
|
||||
handler._serving_runtime_adapter.session_states[session_id] = native
|
||||
|
||||
event = _native_session_create(session_id)
|
||||
event["session"]["extra_body"]["auto_response"] = True
|
||||
@@ -4054,8 +4072,8 @@ async def test_minicpmo_native_duplex_session_close_cleans_auto_response_data_pl
|
||||
|
||||
await handler.handle_session(ws)
|
||||
|
||||
assert not handler._minicpmo_data_plane.has_request(request_id)
|
||||
assert session_id not in handler._minicpmo_sessions
|
||||
assert not handler._serving_runtime_adapter.data_plane.has_request(request_id)
|
||||
assert session_id not in handler._serving_runtime_adapter.session_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -4923,7 +4941,7 @@ async def test_realtime_disconnect_detaches_resumable_session_without_runtime_cl
|
||||
("sid-resumable-disconnect", DuplexLeaseActivity.DETACH),
|
||||
]
|
||||
assert handler._registry.get("sid-resumable-disconnect") is not None
|
||||
assert "sid-resumable-disconnect" in handler._minicpmo_sessions
|
||||
assert "sid-resumable-disconnect" in handler._serving_runtime_adapter.session_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -4947,7 +4965,7 @@ async def test_realtime_initial_resume_token_delivery_failure_closes_unrecoverab
|
||||
|
||||
assert engine.closed == [("sid-created-delivery-failure", "disconnect")]
|
||||
assert handler._registry.get("sid-created-delivery-failure") is None
|
||||
assert "sid-created-delivery-failure" not in handler._minicpmo_sessions
|
||||
assert "sid-created-delivery-failure" not in handler._serving_runtime_adapter.session_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -4983,7 +5001,7 @@ async def test_realtime_send_after_created_disconnect_preserves_resumable_sessio
|
||||
("sid-created-then-disconnected", DuplexLeaseActivity.DETACH),
|
||||
]
|
||||
assert handler._registry.get("sid-created-then-disconnected") is not None
|
||||
assert "sid-created-then-disconnected" in handler._minicpmo_sessions
|
||||
assert "sid-created-then-disconnected" in handler._serving_runtime_adapter.session_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -5007,7 +5025,7 @@ async def test_realtime_resume_rotates_token_replays_and_preserves_runtime_ident
|
||||
second = TimedWebSocket(receive_timeout_s=0.1)
|
||||
second.query_params = {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"minicpmo45_native_duplex": "1",
|
||||
"native_duplex": "1",
|
||||
"resume": "1",
|
||||
}
|
||||
second.put(
|
||||
@@ -5083,7 +5101,7 @@ async def test_realtime_resume_preserves_append_tail_order_across_connections():
|
||||
assert created is not None
|
||||
session = handler._registry.get("sid-resume-append-order")
|
||||
assert session is not None
|
||||
native_state = handler._minicpmo_session_state(session)
|
||||
native_state = handler._runtime_session_state(session)
|
||||
first_scheduler = native_state.silence_continuation_scheduler
|
||||
assert first_scheduler is not None
|
||||
first.put(
|
||||
@@ -5101,7 +5119,7 @@ async def test_realtime_resume_preserves_append_tail_order_across_connections():
|
||||
second = TimedWebSocket(receive_timeout_s=0.5)
|
||||
second.query_params = {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"minicpmo45_native_duplex": "1",
|
||||
"native_duplex": "1",
|
||||
"resume": "1",
|
||||
}
|
||||
second.put(
|
||||
@@ -5402,7 +5420,7 @@ async def test_realtime_disconnect_grace_cancels_only_orphan_response():
|
||||
assert engine.aborted == ["orphan-request"]
|
||||
assert orphan_task.cancelled()
|
||||
assert session.active_response_id is None
|
||||
assert "sid-grace-response" in handler._minicpmo_sessions
|
||||
assert "sid-grace-response" in handler._serving_runtime_adapter.session_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -5453,7 +5471,7 @@ async def test_realtime_idle_ttl_lifecycle_removes_detached_serving_projection()
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
assert handler._registry.get("sid-idle-expired") is None
|
||||
assert "sid-idle-expired" not in handler._minicpmo_sessions
|
||||
assert "sid-idle-expired" not in handler._serving_runtime_adapter.session_states
|
||||
assert "sid-idle-expired" not in handler._session_tasks
|
||||
assert engine.closed == []
|
||||
|
||||
@@ -6428,7 +6446,7 @@ async def test_minicpmo_native_duplex_separates_public_and_runtime_config(monkey
|
||||
return [0.25] * 1600, 16000
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.experimental.fullduplex.minicpmo45.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio",
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.MiniCPMO45NativeDuplexServingAdapter.resolve_ref_audio",
|
||||
fake_resolve_ref_audio,
|
||||
)
|
||||
event = _native_session_create("sid-native-ref-audio", modalities=["text", "audio"])
|
||||
@@ -6554,7 +6572,7 @@ async def test_minicpmo_native_duplex_preserves_ref_audio_channels_until_normali
|
||||
return np.tile(np.array([[0.25, -0.25], [0.5, -0.5]], dtype=np.float32), (800, 1)), 16000
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.experimental.fullduplex.minicpmo45.adapter.MediaConnector",
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.adapter.MediaConnector",
|
||||
FakeMediaConnector,
|
||||
)
|
||||
|
||||
@@ -6580,7 +6598,7 @@ async def test_minicpmo_native_duplex_rejects_ref_audio_path():
|
||||
handler = OmniDuplexSessionHandler(chat_service=chat_service, config_timeout_s=0.1, idle_timeout_s=1)
|
||||
event = _native_session_create("sid-native-ref-path")
|
||||
event["session"]["extra_body"] = {
|
||||
"minicpmo45_native_duplex": True,
|
||||
"native_duplex": True,
|
||||
"ref_audio_path": "/tmp/ref.wav",
|
||||
}
|
||||
ws = TimedWebSocket()
|
||||
@@ -6603,7 +6621,7 @@ def test_minicpmo_native_duplex_explicit_barge_in_request_interrupts():
|
||||
session = DuplexSession(
|
||||
session_id="sid-native-explicit-barge-disabled",
|
||||
config=DuplexSessionConfig(overlap_policy=DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value),
|
||||
capabilities=DuplexCapabilities.minicpmo45_native(),
|
||||
capabilities=minicpmo45_native_capabilities(),
|
||||
)
|
||||
payload = {
|
||||
"type": "audio",
|
||||
@@ -6767,9 +6785,9 @@ async def test_minicpmo_auto_response_restarts_drain_when_append_races_idle_exit
|
||||
session_id="sid-native-late-output",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
session.bind_request(request_id)
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
projected_batches: list[object] = []
|
||||
projected = asyncio.Event()
|
||||
|
||||
@@ -7521,7 +7539,7 @@ async def test_minicpmo_native_auto_response_preserves_silence_continuations_acr
|
||||
session.capabilities.chunk_period_ms = 50
|
||||
session.bind_request(request_id)
|
||||
response_id = session.begin_response(turn_id=session.turn_id)
|
||||
native = handler._minicpmo_session_state(session)
|
||||
native = handler._runtime_session_state(session)
|
||||
scheduler = native.silence_continuation_scheduler
|
||||
assert scheduler is not None
|
||||
|
||||
@@ -7598,7 +7616,7 @@ async def test_minicpmo_native_duplex_uses_segment_text_metadata_for_transcript_
|
||||
|
||||
async def emit(output: object) -> None:
|
||||
result = {"data_plane_outputs": [output]}
|
||||
for native_result in handler._minicpmo_data_plane.project(
|
||||
for native_result in handler._serving_runtime_adapter.data_plane.project(
|
||||
result,
|
||||
context=_data_plane_context(session),
|
||||
):
|
||||
@@ -7682,7 +7700,7 @@ async def test_minicpmo_native_duplex_continuous_speak_reuses_active_response_un
|
||||
|
||||
async def emit(output: object) -> None:
|
||||
result = {"data_plane_outputs": [output]}
|
||||
for native_result in handler._minicpmo_data_plane.project(
|
||||
for native_result in handler._serving_runtime_adapter.data_plane.project(
|
||||
result,
|
||||
context=_data_plane_context(session),
|
||||
):
|
||||
@@ -7932,7 +7950,7 @@ async def test_minicpmo_auto_response_segment_complete_continues_until_model_lis
|
||||
session_id="sid-native-segment-complete",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.capabilities = DuplexCapabilities.minicpmo45_native()
|
||||
session.capabilities = minicpmo45_native_capabilities()
|
||||
response_id = session.begin_response()
|
||||
request_id = "duplex-sid-native-segment-complete-stage0"
|
||||
session.bind_request(request_id)
|
||||
@@ -7968,7 +7986,7 @@ async def test_minicpmo_auto_response_segment_complete_continues_until_model_lis
|
||||
assert emitted is False
|
||||
assert session.active_response_id == response_id
|
||||
assert session.active_request_id == request_id
|
||||
assert not handler._minicpmo_data_plane.is_terminal(request_id)
|
||||
assert not handler._serving_runtime_adapter.data_plane.is_terminal(request_id)
|
||||
assert sent == []
|
||||
assert len(engine.appended) == 1
|
||||
_, mode, payload, final = engine.appended[0]
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.messages import OutputMessage
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.client_request_state import ClientRequestState
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
|
||||
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexOutputAction,
|
||||
DuplexOutputDecision,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision
|
||||
from vllm_omni.experimental.fullduplex.request_client import DuplexRequestClient
|
||||
from vllm_omni.engine.messages import OutputMessage
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.client_request_state import ClientRequestState
|
||||
from vllm_omni.entrypoints.duplex_request_client import DuplexRequestClient
|
||||
from vllm_omni.metrics.stats import OrchestratorAggregator, StageRequestStats, StageStats
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.outputs.duplex import attach_duplex_output_decision
|
||||
|
||||
|
||||
def test_async_omni_uses_extracted_duplex_request_client():
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai_api.test_duplex_handler import FakeChatService, TimedWebSocket
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.openai.protocol import DuplexSession, DuplexSessionConfig
|
||||
from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler
|
||||
from vllm_omni.entrypoints.duplex.protocol import DuplexSession, DuplexSessionConfig
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
Reference in new issue
Block a user