vllm-omni/vllm_omni/experimental/fullduplex/mage_vl/serving/server.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Production WebSocket transport for Mage-VL full-duplex sessions."""

from __future__ import annotations

import argparse
import asyncio
import contextlib
import importlib
import inspect
import json
import secrets
import time
import uuid
from collections.abc import AsyncIterator, Callable
from contextlib import asynccontextmanager
from dataclasses import dataclass
from typing import Any

import uvicorn
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import JSONResponse

from vllm_omni.experimental.fullduplex.core import protocol as event_protocol
from vllm_omni.experimental.fullduplex.core.adapter import DuplexAdapter
from vllm_omni.experimental.fullduplex.core.session import DuplexSession, DuplexSessionConfig
from vllm_omni.experimental.fullduplex.mage_vl import MageVLDuplexAdapter, MageVLDuplexRuntime

AdapterFactory = Callable[[], DuplexAdapter | Any]


@dataclass(frozen=True)
class MageVLServingConfig:
    model: str = "microsoft/Mage-VL"
    max_sessions: int = 32
    idle_timeout_s: float = 300.0
    max_message_bytes: int = 8 * 1024 * 1024
    auth_token: str | None = None

    def __post_init__(self) -> None:
        if self.max_sessions <= 0 or self.idle_timeout_s <= 0 or self.max_message_bytes <= 0:
            raise ValueError("max_sessions, idle_timeout_s, and max_message_bytes must be positive")


class _SessionLease:
    def __init__(self, session_id: str, runtime: MageVLDuplexRuntime) -> None:
        self.session_id = session_id
        self.runtime = runtime
        self.last_activity = time.monotonic()

    def touch(self) -> None:
        self.last_activity = time.monotonic()


class _SessionRegistry:
    def __init__(self, config: MageVLServingConfig, factory: AdapterFactory) -> None:
        self.config = config
        self.factory = factory
        self._leases: dict[str, _SessionLease] = {}
        self._lock = asyncio.Lock()

    async def acquire(self, requested_id: str | None) -> _SessionLease | None:
        async with self._lock:
            if len(self._leases) >= self.config.max_sessions:
                return None
            session_id = requested_id or f"mage-{uuid.uuid4().hex}"
            if not _valid_session_id(session_id) or session_id in self._leases:
                return None
            adapter = self.factory()
            if inspect.isawaitable(adapter):
                adapter = await adapter
            if not isinstance(adapter, MageVLDuplexAdapter):
                raise TypeError("adapter factory must return a MageVLDuplexAdapter")
            capability = adapter.capabilities()
            session = DuplexSession(
                session_id,
                DuplexSessionConfig(
                    input_modalities=tuple(capability.input_modalities),
                    output_modalities=tuple(capability.output_modalities),
                    proactive=capability.proactive,
                ),
            )
            lease = _SessionLease(session_id, MageVLDuplexRuntime(session, adapter))
            self._leases[session_id] = lease
            return lease

    async def release(self, session_id: str) -> None:
        async with self._lock:
            self._leases.pop(session_id, None)

    async def close(self) -> None:
        async with self._lock:
            leases = list(self._leases.values())
            self._leases.clear()
        for lease in leases:
            with contextlib.suppress(asyncio.CancelledError):
                await lease.runtime.close()

    @property
    def active_count(self) -> int:
        return len(self._leases)


def _valid_session_id(value: str) -> bool:
    return 0 < len(value) <= 128 and all(char.isalnum() or char in "-_." for char in value)


def _authorized(websocket: WebSocket, token: str | None) -> bool:
    if token is None:
        return True
    return secrets.compare_digest(websocket.headers.get("authorization", ""), f"Bearer {token}")


def _event_from_text(text: str, max_bytes: int) -> dict[str, Any]:
    if len(text.encode("utf-8")) > max_bytes:
        raise ValueError("message exceeds configured max_message_bytes")
    event = json.loads(text)
    if not isinstance(event, dict) or not isinstance(event.get("type"), str):
        raise ValueError("event must be a JSON object with a string type")
    supported = {
        event_protocol.INPUT_APPEND,
        event_protocol.INPUT_COMMIT,
        event_protocol.RESPONSE_CREATE,
        event_protocol.RESPONSE_CANCEL,
        event_protocol.PLAYBACK_ACK,
        event_protocol.CLOSE,
    }
    if event["type"] not in supported:
        raise ValueError(f"unsupported event type: {event['type']}")
    if event["type"] == event_protocol.INPUT_APPEND and not isinstance(event.get("modality"), str):
        raise ValueError("input.append requires a modality")
    return event


def load_adapter_factory(path: str) -> AdapterFactory:
    module_name, separator, attribute_name = path.partition(":")
    if not separator:
        raise ValueError("adapter path must use module:attribute syntax")
    value = getattr(importlib.import_module(module_name), attribute_name)
    if isinstance(value, DuplexAdapter):
        return lambda: value
    if not callable(value):
        raise TypeError(f"adapter target {path!r} is not callable")
    return value


def create_app(adapter_factory: AdapterFactory, config: MageVLServingConfig | None = None) -> FastAPI:
    config = config or MageVLServingConfig()
    registry = _SessionRegistry(config, adapter_factory)

    @asynccontextmanager
    async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
        try:
            yield
        finally:
            await registry.close()

    app = FastAPI(title="Mage-VL Duplex Server", version="1", lifespan=lifespan)
    app.state.session_registry = registry

    @app.get("/health")
    async def health() -> dict[str, str]:
        return {"status": "ok"}

    @app.get("/ready")
    async def ready() -> JSONResponse:
        return JSONResponse({"status": "ready", "active_sessions": registry.active_count})

    @app.get("/v1/models")
    async def models() -> dict[str, Any]:
        return {"object": "list", "data": [{"id": config.model, "object": "model", "owned_by": "microsoft"}]}

    @app.websocket("/v1/mage-vl/duplex")
    async def duplex(websocket: WebSocket) -> None:
        if not _authorized(websocket, config.auth_token):
            await websocket.close(code=4401, reason="unauthorized")
            return
        await websocket.accept()
        lease = await registry.acquire(websocket.query_params.get("session_id"))
        if lease is None:
            await websocket.close(code=4429, reason="session unavailable")
            return

        async def inputs() -> AsyncIterator[dict[str, Any]]:
            while True:
                event = _event_from_text(
                    await asyncio.wait_for(websocket.receive_text(), config.idle_timeout_s), config.max_message_bytes
                )
                lease.touch()
                yield event
                if event.get("type") == event_protocol.CLOSE:
                    return

        async def emit(event: dict[str, Any]) -> None:
            await websocket.send_json(event)

        capability = lease.runtime.capabilities
        try:
            await websocket.send_json(
                {
                    "type": "session.created",
                    "session_id": lease.session_id,
                    "model": config.model,
                    "capabilities": {
                        "input_modalities": sorted(capability.input_modalities),
                        "output_modalities": sorted(capability.output_modalities),
                        "proactive": capability.proactive,
                    },
                }
            )
            await lease.runtime.run(inputs(), emit)
        except WebSocketDisconnect:
            await lease.runtime.cancel_response()
        except asyncio.TimeoutError:
            with contextlib.suppress(Exception):
                await websocket.send_json({"type": "error", "message": "idle timeout"})
        except (ValueError, json.JSONDecodeError) as error:
            with contextlib.suppress(Exception):
                await websocket.send_json({"type": "error", "message": str(error)})
            await lease.runtime.cancel_response()
        finally:
            await registry.release(lease.session_id)
            with contextlib.suppress(Exception):
                await websocket.close()

    return app


def main() -> None:
    parser = argparse.ArgumentParser(description="Serve Mage-VL full-duplex WebSocket sessions.")
    parser.add_argument("--adapter", help="optional module:attribute adapter factory")
    parser.add_argument("--model", default="microsoft/Mage-VL")
    parser.add_argument("--host", default="127.0.0.1")
    parser.add_argument("--port", type=int, default=8090)
    parser.add_argument("--max-sessions", type=int, default=32)
    parser.add_argument("--idle-timeout", type=float, default=300.0)
    parser.add_argument("--max-message-bytes", type=int, default=8 * 1024 * 1024)
    parser.add_argument("--auth-token")
    parser.add_argument("--device", default="cuda")
    parser.add_argument("--attn-impl", choices=("flash_attention_2", "sdpa", "eager"), default="sdpa")
    parser.add_argument("--video-backend", choices=("frames", "codec"), default="frames")
    parser.add_argument("--num-frames", type=int, default=8)
    parser.add_argument("--target-fps", type=float, default=1.0)
    parser.add_argument("--max-new-tokens", type=int, default=128)
    parser.add_argument("--gate-threshold", type=float, default=0.5)
    parser.add_argument("--window-size", type=int, default=4)
    args = parser.parse_args()
    if args.adapter:
        adapter_factory = load_adapter_factory(args.adapter)
    else:
        from vllm_omni.experimental.fullduplex.mage_vl.serving.backend import MageVLTransformersBackend

        backend = MageVLTransformersBackend(
            args.model,
            device=args.device,
            attn_impl=args.attn_impl,
            video_backend=args.video_backend,
            num_frames=args.num_frames,
            target_fps=args.target_fps,
            max_new_tokens=args.max_new_tokens,
            gate_threshold=args.gate_threshold,
            window_size=args.window_size,
        )
        backend.load()
        adapter_factory = backend.adapter_factory
    app = create_app(
        adapter_factory,
        MageVLServingConfig(
            model=args.model,
            max_sessions=args.max_sessions,
            idle_timeout_s=args.idle_timeout,
            max_message_bytes=args.max_message_bytes,
            auth_token=args.auth_token,
        ),
    )
    uvicorn.run(app, host=args.host, port=args.port)


if __name__ == "__main__":
    main()