[Core] Graduate MiniCPM-o 4.5 and PersonaPlex full-duplex serving out of experimental (#6196)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>

vllm_omni/benchmarks/patch/patch.py

@@ -18,7 +18,7 @@ from collections.abc import Iterable
 from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 import aiohttp
 import numpy as np
@@ -70,6 +70,7 @@ from vllm_omni.benchmarks.omniinteract import (
     VIDEO_FPS,
     OmniInteractBenchmarkConfig,
     OmniInteractCaseResult,
+    _audio_rtf_from_raw_metric,
     clear_batch_artifacts,
     clear_case_artifacts,
     prepare_media,
@@ -83,16 +84,18 @@ from vllm_omni.benchmarks.omniinteract import (
 from vllm_omni.benchmarks.omniinteract import (
     write_batch_artifacts as write_omniinteract_batch_artifacts,
 )
-from vllm_omni.experimental.fullduplex.client import (
-    RealtimeDuplexClient,
-    build_realtime_url,
-    reference_audio_data_url,
-    summarize_session_request_metrics,
-    wait_for,
-)
 from vllm_omni.metrics import definitions as defs
 from vllm_omni.metrics.utils import coerce_positive_int_scalar
 
+if TYPE_CHECKING:
+    from vllm_omni.clients.duplex import DuplexClient
+
+# The duplex client library (vllm_omni.clients) is imported lazily inside the
+# functions that drive Realtime sessions: this module is imported for its
+# patching side effects by the CLI package (`vllm-omni serve` included), and
+# the client package must stay out of that import graph
+# (tests/engine/test_duplex_import_boundary.py enforces the boundary).
+
 logger = init_logger(__name__)
 
 _AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S"
@@ -423,6 +426,8 @@ def get_samples(args, tokenizer):
         )
         if not requests:
             raise ValueError("No OmniInteract sessions were selected")
+        from vllm_omni.clients.duplex import reference_audio_data_url
+
         encoded_ref_audio = reference_audio_data_url(options.ref_audio)
         assert encoded_ref_audio is not None
         for request in requests:
@@ -1531,6 +1536,8 @@ async def async_request_openai_audio_speech(
 
 
 def _realtime_websocket_url(api_url: str) -> str:
+    from vllm_omni.clients.duplex import build_realtime_url
+
     return build_realtime_url(api_url, None, native_duplex=None)
 
 
@@ -1685,11 +1692,93 @@ async def _async_request_omniinteract(
     return output
 
 
+class _RealtimeTTSProbe:
+    """Explicit-session Realtime TTS driver over the public duplex client.
+
+    Owns an :class:`EventCollector` (``.events``) so turn metrics can be read
+    with controlled timestamps; the session handshake happens in
+    :meth:`configure` because the benchmark decides the session shape at
+    request time.
+    """
+
+    def __init__(self, url: str) -> None:
+        from vllm_omni.clients.duplex import EventCollector
+
+        self._url = url
+        self.events: EventCollector = EventCollector()
+        self._client: DuplexClient | None = None
+        self._consume_task: asyncio.Task[None] | None = None
+
+    async def __aenter__(self) -> "_RealtimeTTSProbe":
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        if self._client is not None:
+            await self._client.__aexit__(exc_type, exc, tb)
+        if self._consume_task is not None:
+            try:
+                await asyncio.wait_for(self._consume_task, timeout=5.0)
+            except (asyncio.TimeoutError, asyncio.CancelledError):
+                self._consume_task.cancel()
+
+    async def configure(
+        self,
+        model: str,
+        *,
+        output_audio_format: str = "pcm16",
+        instructions: str | None = None,
+        native_duplex: bool = False,
+        auto_response: bool = False,
+        extra_body: dict[str, object] | None = None,
+        session_id: str | None = None,
+        timeout_s: float = 120.0,
+    ) -> None:
+        from vllm_omni.clients.duplex import AudioFormat, DuplexClient, SessionConfig
+
+        session_extra_body: dict[str, object] = dict(extra_body or {})
+        session_extra_body["native_duplex"] = bool(native_duplex)
+        config = SessionConfig(
+            output_audio=AudioFormat(output_audio_format, 24_000),
+            instructions=instructions,
+            auto_response=auto_response,
+            overlap_policy="listen_only",
+            playback_commit_policy="ack_only",
+            extra_body=session_extra_body,
+        )
+        self._client = DuplexClient(
+            self._url,
+            model=model,
+            config=config,
+            session_id=session_id,
+            reconnect=None,
+            heartbeat_interval_s=None,
+            handshake_timeout_s=timeout_s,
+        )
+        await self._client.__aenter__()
+        self._consume_task = asyncio.create_task(self.events.consume(self._client))
+
+    async def send(self, event: dict[str, object]) -> None:
+        assert self._client is not None
+        await self._client.send(event)
+
+    async def acknowledge_playback(self) -> None:
+        from vllm_omni.clients.duplex import acknowledge_collected_playback
+
+        assert self._client is not None
+        await acknowledge_collected_playback(self._client, self.events)
+
+    async def close_session(self, *, timeout_s: float = 20.0) -> None:
+        assert self._client is not None
+        await self._client.close(timeout_s=timeout_s)
+
+
 async def async_request_openai_realtime_duplex(
     request_func_input: RequestFuncInput,
     session: aiohttp.ClientSession,
     pbar: tqdm | None = None,
 ) -> MixRequestFuncOutput:
+    from vllm_omni.clients.duplex import summarize_session_request_metrics, wait_for_condition
+
     del session
     if getattr(request_func_input, "omniinteract_case", None) is not None:
         return await _async_request_omniinteract(request_func_input, pbar=pbar)
@@ -1715,7 +1804,7 @@ async def async_request_openai_realtime_duplex(
         turn_prompts = [("", request_func_input.prompt)]
     session_id = f"seed-tts-{request_func_input.request_id or uuid.uuid4().hex}"
     try:
-        async with RealtimeDuplexClient(_realtime_websocket_url(request_func_input.api_url)) as client:
+        async with _RealtimeTTSProbe(_realtime_websocket_url(request_func_input.api_url)) as client:
             await client.configure(
                 request_func_input.model_name or request_func_input.model,
                 output_audio_format="pcm16",
@@ -1755,7 +1844,7 @@ async def async_request_openai_realtime_duplex(
                     }
                 )
                 await client.send({"type": "response.create"})
-                await wait_for(
+                await wait_for_condition(
                     lambda: (
                         client.events.count("response.done") > done_before
                         or len(client.events.errors()) > errors_before
@@ -1794,6 +1883,9 @@ async def async_request_openai_realtime_duplex(
                         "utterance_id": utterance_id or None,
                         "response_id": response_id,
                         **request_metrics,
+                        # The client reports raw data only; the RTF is derived
+                        # here with the canonical server-side definition.
+                        "rtf": _audio_rtf_from_raw_metric(request_metrics),
                     }
                 )
                 response_audio = client.events.audio_bytes(response_id)