[Perf][NemotronVoiceChat] Fast defaults for all three deploy profiles: 2.8x offline, realtime full-duplex audio (#6354)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e43c787544
commit
4179b0cae1
22 files changed
+2606
-195
No files matched your search
@@ -13,6 +13,7 @@ import os
|
||||
import random
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
@@ -445,6 +446,116 @@ class _DiffusionServingModels:
|
||||
# Server entry points
|
||||
|
||||
|
||||
async def _warmup_duplex_realtime(app, args, warmup_frames: int) -> None:
|
||||
"""Run silent frames through a throwaway realtime session at startup.
|
||||
|
||||
Self-connects over the server's own websocket endpoint so the warmup
|
||||
exercises the exact production path (serving adapter, duplex control
|
||||
plane, every pipeline stage). One-time costs — kernel JIT compilation,
|
||||
first prefill/decode paths, codec caches — are paid here instead of by
|
||||
the first real client; ``/v1/realtime`` holds other connections until
|
||||
the warmup finishes (see ``duplex_warmup_done``).
|
||||
"""
|
||||
import base64
|
||||
|
||||
warmup_done = getattr(app.state, "duplex_warmup_done", None)
|
||||
try:
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
logger.warning("Duplex warmup skipped: the 'websockets' package is not installed.")
|
||||
return
|
||||
served = getattr(args, "served_model_name", None)
|
||||
if isinstance(served, list | tuple) and served:
|
||||
model_name = served[0]
|
||||
elif isinstance(served, str) and served:
|
||||
model_name = served
|
||||
else:
|
||||
model_name = args.model
|
||||
adapter = getattr(getattr(app.state, "openai_serving_duplex", None), "_serving_runtime_adapter", None)
|
||||
frame_samples = int(getattr(adapter, "silence_continuation_samples", 16000))
|
||||
silence = base64.b64encode(bytes(frame_samples * 4)).decode("ascii")
|
||||
from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
||||
|
||||
url = (
|
||||
build_realtime_url(f"ws://127.0.0.1:{args.port}/v1/realtime", model_name, autostart=False)
|
||||
+ "&vllm_omni_warmup=1"
|
||||
)
|
||||
logger.info("Duplex warmup starting: %d silent frames of %d samples.", warmup_frames, frame_samples)
|
||||
start = time.time()
|
||||
# The socket is bound before uvicorn finishes starting, so a plain
|
||||
# connect can be accepted and then reset. Wait for /health first.
|
||||
async with httpx.AsyncClient() as http:
|
||||
for _ in range(120):
|
||||
try:
|
||||
if (await http.get(f"http://127.0.0.1:{args.port}/health", timeout=2)).status_code == 200:
|
||||
break
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
logger.warning("Duplex warmup: /health never became ready; skipping warmup.")
|
||||
return
|
||||
async with websockets.connect(url, max_size=64 * 1024 * 1024, open_timeout=10) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": f"warmup-{uuid.uuid4().hex[:8]}",
|
||||
"model": model_name,
|
||||
"modalities": ["audio", "text"],
|
||||
"input_audio_format": "pcm_f32le",
|
||||
"output_audio_format": "pcm16",
|
||||
"idle_timeout_s": 60,
|
||||
"turn_detection": None,
|
||||
"extra_body": {"auto_response": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
saw_audio = False
|
||||
sent = 0
|
||||
|
||||
async def _recv_until(predicate, timeout_s: float) -> bool:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
raw = await asyncio.wait_for(ws.recv(), timeout=max(0.05, deadline - time.monotonic()))
|
||||
except (TimeoutError, asyncio.TimeoutError):
|
||||
return False
|
||||
event = json.loads(raw)
|
||||
if predicate(event):
|
||||
return True
|
||||
return False
|
||||
|
||||
if not await _recv_until(lambda e: e.get("type") == "session.created", 30):
|
||||
logger.warning("Duplex warmup: no session.created within 30 s; aborting warmup.")
|
||||
return
|
||||
while sent < warmup_frames:
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": silence}))
|
||||
sent += 1
|
||||
await asyncio.sleep(0.08)
|
||||
# Wait for the pipeline's first audio output so every stage ran.
|
||||
saw_audio = await _recv_until(lambda e: e.get("type") == "response.audio.delta", 30)
|
||||
# Close the session EXPLICITLY: a bare websocket close parks the
|
||||
# session in its disconnect grace window, where it would keep
|
||||
# occupying the max_sessions=1 slot and block the first client.
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
await _recv_until(lambda e: e.get("type") == "session.closed", 10)
|
||||
logger.info(
|
||||
"Duplex warmup finished in %.1f s (%d silent frames, first audio %s).",
|
||||
time.time() - start,
|
||||
sent,
|
||||
"received" if saw_audio else "NOT received",
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Duplex warmup failed; continuing to serve (first session pays cold-start costs).")
|
||||
finally:
|
||||
if warmup_done is not None:
|
||||
warmup_done.set()
|
||||
|
||||
|
||||
async def omni_run_server(args, **uvicorn_kwargs) -> None:
|
||||
"""Run a single-worker API server.
|
||||
|
||||
@@ -585,6 +696,21 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
scope["state"]["request_timestamp"] = time.time()
|
||||
await self._inner(scope, receive, send)
|
||||
|
||||
# Startup duplex warmup (duplex_session.warmup_frames in the deploy
|
||||
# yaml): real /v1/realtime connections wait on this event so the
|
||||
# first client never pays cold-start costs.
|
||||
duplex_warmup_frames = 0
|
||||
if getattr(app.state, "openai_serving_duplex", None) is not None:
|
||||
duplex_cfg = getattr(engine_client, "duplex_session_config", None)
|
||||
duplex_warmup_frames = int(getattr(duplex_cfg, "warmup_frames", 0) or 0)
|
||||
app.state.duplex_warmup_done = asyncio.Event() if duplex_warmup_frames > 0 else None
|
||||
warmup_task: asyncio.Task | None = None
|
||||
if duplex_warmup_frames > 0:
|
||||
# Scheduled before serve_http (which may not return until
|
||||
# shutdown); the coroutine retries its self-connect until the
|
||||
# server socket is accepting.
|
||||
warmup_task = asyncio.create_task(_warmup_duplex_realtime(app, args, duplex_warmup_frames))
|
||||
|
||||
shutdown_task = await serve_http(
|
||||
_TimestampMiddleware(app),
|
||||
sock=sock,
|
||||
@@ -609,6 +735,8 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
try:
|
||||
await shutdown_task
|
||||
finally:
|
||||
if warmup_task is not None:
|
||||
warmup_task.cancel()
|
||||
state = getattr(app, "state", None)
|
||||
serving_video = getattr(state, "openai_serving_video", None) if state is not None else None
|
||||
if serving_video is not None:
|
||||
@@ -1709,6 +1837,14 @@ async def streaming_video_output(websocket: WebSocket):
|
||||
@router.websocket("/v1/realtime")
|
||||
async def realtime_websocket(websocket: WebSocket):
|
||||
"""WebSocket endpoint for OpenAI-style realtime interactions."""
|
||||
# Hold real clients until the startup duplex warmup finishes (the warmup
|
||||
# connection marks itself with vllm_omni_warmup=1 and passes through).
|
||||
warmup_done = getattr(websocket.app.state, "duplex_warmup_done", None)
|
||||
if warmup_done is not None and not warmup_done.is_set() and websocket.query_params.get("vllm_omni_warmup") != "1":
|
||||
try:
|
||||
await asyncio.wait_for(warmup_done.wait(), timeout=120)
|
||||
except (TimeoutError, asyncio.TimeoutError):
|
||||
logger.warning("Duplex warmup still running after 120 s; admitting the client anyway.")
|
||||
duplex_handler = getattr(websocket.app.state, "openai_serving_duplex", None)
|
||||
duplex_query = websocket.query_params.get("duplex")
|
||||
use_duplex_realtime = (
|
||||
|
||||
Reference in new issue
Block a user