[Perf][NemotronVoiceChat] Fast defaults for all three deploy profiles: 2.8x offline, realtime full-duplex audio (#6354)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yuekai ZhangandClaude Fable 5 authored and GitHub committed 2026-09-03 18:49:19 -07:00
1 parent e43c787544
commit 4179b0cae1
22 files changed
+2606 -195

No files matched your search

+136
View File
@@ -13,6 +13,7 @@ import os
import random
import tempfile
import time
import uuid
from argparse import Namespace
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
@@ -445,6 +446,116 @@ class _DiffusionServingModels:
# Server entry points
async def _warmup_duplex_realtime(app, args, warmup_frames: int) -> None:
"""Run silent frames through a throwaway realtime session at startup.
Self-connects over the server's own websocket endpoint so the warmup
exercises the exact production path (serving adapter, duplex control
plane, every pipeline stage). One-time costs — kernel JIT compilation,
first prefill/decode paths, codec caches — are paid here instead of by
the first real client; ``/v1/realtime`` holds other connections until
the warmup finishes (see ``duplex_warmup_done``).
"""
import base64
warmup_done = getattr(app.state, "duplex_warmup_done", None)
try:
try:
import websockets
except ImportError:
logger.warning("Duplex warmup skipped: the 'websockets' package is not installed.")
return
served = getattr(args, "served_model_name", None)
if isinstance(served, list | tuple) and served:
model_name = served[0]
elif isinstance(served, str) and served:
model_name = served
else:
model_name = args.model
adapter = getattr(getattr(app.state, "openai_serving_duplex", None), "_serving_runtime_adapter", None)
frame_samples = int(getattr(adapter, "silence_continuation_samples", 16000))
silence = base64.b64encode(bytes(frame_samples * 4)).decode("ascii")
from vllm_omni.experimental.fullduplex.client import build_realtime_url
url = (
build_realtime_url(f"ws://127.0.0.1:{args.port}/v1/realtime", model_name, autostart=False)
+ "&vllm_omni_warmup=1"
)
logger.info("Duplex warmup starting: %d silent frames of %d samples.", warmup_frames, frame_samples)
start = time.time()
# The socket is bound before uvicorn finishes starting, so a plain
# connect can be accepted and then reset. Wait for /health first.
async with httpx.AsyncClient() as http:
for _ in range(120):
try:
if (await http.get(f"http://127.0.0.1:{args.port}/health", timeout=2)).status_code == 200:
break
except httpx.HTTPError:
pass
await asyncio.sleep(1)
else:
logger.warning("Duplex warmup: /health never became ready; skipping warmup.")
return
async with websockets.connect(url, max_size=64 * 1024 * 1024, open_timeout=10) as ws:
await ws.send(
json.dumps(
{
"type": "session.update",
"session": {
"session_id": f"warmup-{uuid.uuid4().hex[:8]}",
"model": model_name,
"modalities": ["audio", "text"],
"input_audio_format": "pcm_f32le",
"output_audio_format": "pcm16",
"idle_timeout_s": 60,
"turn_detection": None,
"extra_body": {"auto_response": True},
},
}
)
)
saw_audio = False
sent = 0
async def _recv_until(predicate, timeout_s: float) -> bool:
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
try:
raw = await asyncio.wait_for(ws.recv(), timeout=max(0.05, deadline - time.monotonic()))
except (TimeoutError, asyncio.TimeoutError):
return False
event = json.loads(raw)
if predicate(event):
return True
return False
if not await _recv_until(lambda e: e.get("type") == "session.created", 30):
logger.warning("Duplex warmup: no session.created within 30 s; aborting warmup.")
return
while sent < warmup_frames:
await ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": silence}))
sent += 1
await asyncio.sleep(0.08)
# Wait for the pipeline's first audio output so every stage ran.
saw_audio = await _recv_until(lambda e: e.get("type") == "response.audio.delta", 30)
# Close the session EXPLICITLY: a bare websocket close parks the
# session in its disconnect grace window, where it would keep
# occupying the max_sessions=1 slot and block the first client.
await ws.send(json.dumps({"type": "session.close"}))
await _recv_until(lambda e: e.get("type") == "session.closed", 10)
logger.info(
"Duplex warmup finished in %.1f s (%d silent frames, first audio %s).",
time.time() - start,
sent,
"received" if saw_audio else "NOT received",
)
except Exception:
logger.exception("Duplex warmup failed; continuing to serve (first session pays cold-start costs).")
finally:
if warmup_done is not None:
warmup_done.set()
async def omni_run_server(args, **uvicorn_kwargs) -> None:
"""Run a single-worker API server.
@@ -585,6 +696,21 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
scope["state"]["request_timestamp"] = time.time()
await self._inner(scope, receive, send)
# Startup duplex warmup (duplex_session.warmup_frames in the deploy
# yaml): real /v1/realtime connections wait on this event so the
# first client never pays cold-start costs.
duplex_warmup_frames = 0
if getattr(app.state, "openai_serving_duplex", None) is not None:
duplex_cfg = getattr(engine_client, "duplex_session_config", None)
duplex_warmup_frames = int(getattr(duplex_cfg, "warmup_frames", 0) or 0)
app.state.duplex_warmup_done = asyncio.Event() if duplex_warmup_frames > 0 else None
warmup_task: asyncio.Task | None = None
if duplex_warmup_frames > 0:
# Scheduled before serve_http (which may not return until
# shutdown); the coroutine retries its self-connect until the
# server socket is accepting.
warmup_task = asyncio.create_task(_warmup_duplex_realtime(app, args, duplex_warmup_frames))
shutdown_task = await serve_http(
_TimestampMiddleware(app),
sock=sock,
@@ -609,6 +735,8 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
try:
await shutdown_task
finally:
if warmup_task is not None:
warmup_task.cancel()
state = getattr(app, "state", None)
serving_video = getattr(state, "openai_serving_video", None) if state is not None else None
if serving_video is not None:
@@ -1709,6 +1837,14 @@ async def streaming_video_output(websocket: WebSocket):
@router.websocket("/v1/realtime")
async def realtime_websocket(websocket: WebSocket):
"""WebSocket endpoint for OpenAI-style realtime interactions."""
# Hold real clients until the startup duplex warmup finishes (the warmup
# connection marks itself with vllm_omni_warmup=1 and passes through).
warmup_done = getattr(websocket.app.state, "duplex_warmup_done", None)
if warmup_done is not None and not warmup_done.is_set() and websocket.query_params.get("vllm_omni_warmup") != "1":
try:
await asyncio.wait_for(warmup_done.wait(), timeout=120)
except (TimeoutError, asyncio.TimeoutError):
logger.warning("Duplex warmup still running after 120 s; admitting the client anyway.")
duplex_handler = getattr(websocket.app.state, "openai_serving_duplex", None)
duplex_query = websocket.query_params.get("duplex")
use_duplex_realtime = (