vllm-omni/vllm_omni/entrypoints/duplex/warmup.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Startup warmup probe for duplex ``/v1/realtime``.

Moved out of ``api_server.py`` under P0.2 of #5227. The
``omni_run_server_worker`` scheduling and ``/v1/realtime`` hold-clients
preamble stay in ``api_server.py`` until P0.3.
"""

import asyncio
import json
import time
import uuid

import httpx
import pybase64 as base64
from vllm.logger import init_logger

logger = init_logger(__name__)


async def _warmup_duplex_realtime(app, args, warmup_frames: int) -> None:
    """Run silent frames through a throwaway realtime session at startup.

    Self-connects over the server's own websocket endpoint so the warmup
    exercises the exact production path (serving adapter, duplex control
    plane, every pipeline stage). One-time costs — kernel JIT compilation,
    first prefill/decode paths, codec caches — are paid here instead of by
    the first real client; ``/v1/realtime`` holds other connections until
    the warmup finishes (see ``duplex_warmup_done``).
    """
    warmup_done = getattr(app.state, "duplex_warmup_done", None)
    try:
        try:
            import websockets
        except ImportError:
            logger.warning("Duplex warmup skipped: the 'websockets' package is not installed.")
            return
        served = getattr(args, "served_model_name", None)
        if isinstance(served, list | tuple) and served:
            model_name = served[0]
        elif isinstance(served, str) and served:
            model_name = served
        else:
            model_name = args.model
        # One silence unit as the engine-side plugin defines it (DuplexOmniEngine.plugin).
        plugin = getattr(getattr(app.state.engine_client, "engine", None), "plugin", None)
        frame_samples = int(getattr(plugin, "silence_continuation_samples", 16000))
        silence = base64.b64encode(bytes(frame_samples * 4)).decode("ascii")
        from vllm_omni.clients.duplex import build_realtime_url

        url = (
            build_realtime_url(f"ws://127.0.0.1:{args.port}/v1/realtime", model_name, autostart=False)
            + "&vllm_omni_warmup=1"
        )
        logger.info("Duplex warmup starting: %d silent frames of %d samples.", warmup_frames, frame_samples)
        start = time.time()
        # The socket is bound before uvicorn finishes starting, so a plain
        # connect can be accepted and then reset. Wait for /health first.
        async with httpx.AsyncClient() as http:
            for _ in range(120):
                try:
                    if (await http.get(f"http://127.0.0.1:{args.port}/health", timeout=2)).status_code == 200:
                        break
                except httpx.HTTPError:
                    pass
                await asyncio.sleep(1)
            else:
                logger.warning("Duplex warmup: /health never became ready; skipping warmup.")
                return
        async with websockets.connect(url, max_size=64 * 1024 * 1024, open_timeout=10) as ws:
            await ws.send(
                json.dumps(
                    {
                        "type": "session.update",
                        "session": {
                            "session_id": f"warmup-{uuid.uuid4().hex[:8]}",
                            "model": model_name,
                            "modalities": ["audio", "text"],
                            "input_audio_format": "pcm_f32le",
                            "output_audio_format": "pcm16",
                            "idle_timeout_s": 60,
                            "turn_detection": None,
                            "extra_body": {"auto_response": True},
                        },
                    }
                )
            )
            saw_audio = False
            sent = 0

            async def _recv_until(predicate, timeout_s: float) -> bool:
                deadline = time.monotonic() + timeout_s
                while time.monotonic() < deadline:
                    try:
                        raw = await asyncio.wait_for(ws.recv(), timeout=max(0.05, deadline - time.monotonic()))
                    except (TimeoutError, asyncio.TimeoutError):
                        return False
                    event = json.loads(raw)
                    if predicate(event):
                        return True
                return False

            if not await _recv_until(lambda e: e.get("type") == "session.created", 30):
                logger.warning("Duplex warmup: no session.created within 30 s; aborting warmup.")
                return
            while sent < warmup_frames:
                await ws.send(json.dumps({"type": "input_audio_buffer.append", "audio": silence}))
                sent += 1
                await asyncio.sleep(0.08)
            # Wait for the pipeline's first audio output so every stage ran.
            saw_audio = await _recv_until(lambda e: e.get("type") == "response.output_audio.delta", 30)
            # Close the session EXPLICITLY: a bare websocket close parks the
            # session in its disconnect grace window, where it would keep
            # occupying the max_sessions=1 slot and block the first client.
            await ws.send(json.dumps({"type": "session.close"}))
            await _recv_until(lambda e: e.get("type") == "session.closed", 10)
        logger.info(
            "Duplex warmup finished in %.1f s (%d silent frames, first audio %s).",
            time.time() - start,
            sent,
            "received" if saw_audio else "NOT received",
        )
    except Exception:
        logger.exception("Duplex warmup failed; continuing to serve (first session pays cold-start costs).")
    finally:
        if warmup_done is not None:
            warmup_done.set()


__all__ = ["_warmup_duplex_realtime"]