Rebase multi-API serving on latest main

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 committed 2026-09-16 12:13:04 +08:00
1 parent 78934753ff
commit dc3c7dbe16
25 files changed
+1832 -202

No files matched your search

+129 -7
View File
@@ -6,6 +6,7 @@ This module owns app construction, server startup, app-state initialization,
and route bodies that have not yet moved to endpoint-owned modules."""
import asyncio
import copy
import dataclasses
import json
import multiprocessing
@@ -14,6 +15,8 @@ import os
# Image generation API imports
import random
import signal
import socket
import time
from argparse import Namespace
from collections.abc import AsyncIterator
@@ -21,6 +24,7 @@ from contextlib import asynccontextmanager
from http import HTTPStatus
from typing import Annotated, Any, Literal
import uvloop
import vllm.envs as envs
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile, WebSocket
from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse
@@ -80,12 +84,14 @@ from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import POOLING_TASKS
from vllm.tool_parsers import ToolParserManager
from vllm.utils import random_uuid
from vllm.utils.system_utils import decorate_logs
from vllm.utils.system_utils import decorate_logs, set_process_title
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.config.endpoint_policy import (
shutdown_unsupported_routes,
)
from vllm_omni.engine.stage_init_utils import set_death_signal
from vllm_omni.engine.stage_runtime import OmniClientConfig
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.duplex.capability import should_enable_duplex_endpoint
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
@@ -229,8 +235,55 @@ async def omni_run_server(args, **uvicorn_kwargs) -> None:
await omni_run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
async def omni_run_server_worker(listen_address, sock, args, client_config=None, **uvicorn_kwargs) -> None:
def run_omni_api_server_worker_proc(
listen_address: str,
sock: socket.socket,
args: TrackingNamespace,
client_config: dict[str, Any] | None = None,
**uvicorn_kwargs: object,
) -> None:
"""Entrypoint used by vLLM's API server process manager."""
set_death_signal(signal.SIGTERM)
manager_config = client_config or {}
client_index = int(manager_config.get("client_index", 0))
all_client_configs = getattr(args, "_omni_stage_client_configs", None)
if not all_client_configs or not 0 <= client_index < len(all_client_configs):
raise RuntimeError(f"Missing Omni stage client configuration for API server {client_index}")
omni_client_config: OmniClientConfig = copy.deepcopy(all_client_configs[client_index])
omni_client_config["client_count"] = int(manager_config.get("client_count", 1))
omni_client_config["client_index"] = client_index
stage_addresses = omni_client_config["stage_addresses"]
first_stage_id = min(stage_addresses)
first_replica_id = min(stage_addresses[first_stage_id])
first_addresses = stage_addresses[first_stage_id][first_replica_id]
for key in ("input_address", "output_address", "actual_address_pipe", "tensor_queue"):
if key in manager_config:
first_addresses[key] = manager_config[key]
set_process_title("APIServer", str(client_index))
decorate_logs("APIServer", skip_if_decorated=True)
uvloop.run(
omni_run_server_worker(
listen_address,
sock,
args,
client_config=omni_client_config,
**uvicorn_kwargs,
)
)
async def omni_run_server_worker(
listen_address: str,
sock: socket.socket,
args: TrackingNamespace,
client_config: OmniClientConfig | None = None,
**uvicorn_kwargs: object,
) -> None:
"""Run a single API server worker."""
api_server_count = _resolve_api_server_count(args, client_config)
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
@@ -270,6 +323,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
# OMNI: Pass supported_tasks to build_app (required by upstream vLLM)
app = build_openai_app(args, supported_tasks)
app.state.api_server_count = api_server_count
# OMNI: Remove upstream routes that we override with omni-specific handlers
remove_route_from_app(app, "/v1/chat/completions", {"POST"})
@@ -345,6 +399,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
if scope["type"] == "http":
scope.setdefault("state", {})
scope["state"]["request_timestamp"] = time.time()
await self._inner(scope, receive, send)
# Startup duplex warmup (duplex_session.warmup_frames in the deploy
@@ -403,7 +458,7 @@ async def build_async_omni(
args: TrackingNamespace,
*,
disable_frontend_multiprocessing: bool | None = None,
client_config: dict[str, Any] | None = None,
client_config: OmniClientConfig | None = None,
) -> AsyncIterator[EngineClient]:
"""Build an AsyncOmni instance from command-line arguments.
@@ -434,6 +489,7 @@ async def build_async_omni(
async with build_async_omni_from_stage_config(
args,
disable_frontend_multiprocessing=disable_frontend_multiprocessing,
client_config=client_config,
) as async_omni:
yield async_omni
@@ -443,6 +499,7 @@ async def build_async_omni_from_stage_config(
args: TrackingNamespace,
*,
disable_frontend_multiprocessing: bool = False,
client_config: OmniClientConfig | None = None,
) -> AsyncIterator[EngineClient]:
"""Create AsyncOmni from stage configuration.
@@ -499,6 +556,8 @@ async def build_async_omni_from_stage_config(
kwargs.pop("robot_openpi_idle_timeout", None)
model = kwargs.pop("model", None) or args.model
kwargs.setdefault("log_stats", not args.disable_log_stats)
if client_config is not None:
kwargs["client_config"] = client_config
async_omni = AsyncOmni(model=model, **kwargs)
# # Don't keep the dummy data in memory
@@ -1272,6 +1331,42 @@ async def list_voices(raw_request: Request):
return JSONResponse(content={"voices": speakers, "uploaded_voices": uploaded_speakers})
def _resolve_api_server_count(args: Namespace, client_config: OmniClientConfig | None) -> int:
"""Resolve frontend topology once, from the launcher's client configuration."""
requested = getattr(args, "api_server_count", None)
actual = 1 if client_config is None else client_config.get("client_count")
if type(actual) is not int or actual < 1:
raise ValueError("API client configuration requires a positive integer client_count")
if requested is not None and (type(requested) is not int or requested < 1 or requested != actual):
raise ValueError("API server count does not match the frontend launch configuration")
return actual
def _reject_process_local_state_with_multiple_api_workers(raw_request: Request, operation: str) -> None:
"""Reject access to frontend state that is not shared across API workers."""
api_server_count = getattr(raw_request.app.state, "api_server_count", None)
if type(api_server_count) is not int or api_server_count < 1:
raise HTTPException(
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
detail="API worker topology is not initialized.",
)
if api_server_count > 1:
raise HTTPException(
status_code=HTTPStatus.CONFLICT.value,
detail=(
f"{operation} is not supported with --api-server-count > 1 because "
"the operation uses process-local frontend state"
),
)
async def _require_single_api_video_store(raw_request: Request) -> None:
# VIDEO_STORE and VIDEO_TASKS are process-local, including reads and
# cancellation. Keep this independent of the temporary diffusion launch
# restriction: enabling multi-API diffusion alone cannot make jobs shared.
_reject_process_local_state_with_multiple_api_workers(raw_request, "Asynchronous video jobs")
@router.post(
"/v1/audio/voices",
responses={
@@ -1317,6 +1412,7 @@ async def upload_voice(
Returns:
JSON response with voice information
"""
_reject_process_local_state_with_multiple_api_workers(raw_request, "Runtime voice upload")
handler = Omnispeech(raw_request)
if handler is None:
return _create_speech_error_json_response(
@@ -1382,6 +1478,7 @@ async def delete_voice(name: str, raw_request: Request):
Returns:
JSON response indicating success or failure
"""
_reject_process_local_state_with_multiple_api_workers(raw_request, "Runtime voice deletion")
handler = Omnispeech(raw_request)
if handler is None:
return _create_speech_error_json_response(
@@ -1489,6 +1586,8 @@ async def realtime_websocket(websocket: WebSocket):
duplex_query is None or (isinstance(duplex_query, str) and duplex_query.lower() in {"1", "true", "on"})
)
if use_duplex_realtime and duplex_handler is not None:
if await _reject_multi_api_duplex(websocket):
return
await duplex_handler.handle_realtime_session(websocket)
return
@@ -1535,6 +1634,8 @@ async def realtime_robot_openpi(websocket: WebSocket):
@router.websocket("/v1/duplex")
async def duplex_websocket(websocket: WebSocket):
"""WebSocket endpoint for vLLM-Omni duplex session control."""
if await _reject_multi_api_duplex(websocket):
return
handler = getattr(websocket.app.state, "openai_serving_duplex", None)
if handler is None:
await websocket.accept()
@@ -1544,6 +1645,24 @@ async def duplex_websocket(websocket: WebSocket):
await handler.handle_session(websocket)
async def _reject_multi_api_duplex(websocket: WebSocket) -> bool:
# Session credentials and replay state are local to each frontend; a shared
# listening socket cannot route reconnects back to the session owner.
count = getattr(websocket.app.state, "api_server_count", None)
if count is None or (type(count) is int and count == 1):
return False
await websocket.accept()
await websocket.send_json(
{
"type": "error",
"error": "Duplex sessions require a single API worker with initialized topology.",
"code": "multi_api_duplex_unsupported",
}
)
await websocket.close(code=1008)
return True
# RL Rollout serving (RFC #3747, P0)
@@ -2311,6 +2430,7 @@ async def edit_images(
@router.post(
"/v1/videos",
dependencies=[Depends(_require_single_api_video_store)],
responses={
HTTPStatus.OK.value: {"model": VideoResponse},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
@@ -2452,7 +2572,7 @@ async def create_video_sync(
)
@router.get("/v1/videos", response_model=VideoListResponse)
@router.get("/v1/videos", response_model=VideoListResponse, dependencies=[Depends(_require_single_api_video_store)])
async def list_videos(
after: str | None = None,
limit: int | None = Query(None, ge=0, le=100),
@@ -2489,7 +2609,7 @@ async def list_videos(
return VideoListResponse(data=jobs, has_more=has_more, first_id=first_id, last_id=last_id)
@router.get("/v1/videos/{video_id}", response_model=None)
@router.get("/v1/videos/{video_id}", response_model=None, dependencies=[Depends(_require_single_api_video_store)])
async def retrieve_video(video_id: str) -> VideoResponse | JSONResponse:
"""Retrieve metadata for a previously created video job.
@@ -2517,7 +2637,7 @@ async def retrieve_video(video_id: str) -> VideoResponse | JSONResponse:
return job
@router.delete("/v1/videos/{video_id}")
@router.delete("/v1/videos/{video_id}", dependencies=[Depends(_require_single_api_video_store)])
async def delete_video(video_id: str) -> VideoDeleteResponse:
"""Delete a stored video job and any generated output.
@@ -2569,7 +2689,7 @@ async def delete_video(video_id: str) -> VideoDeleteResponse:
return VideoDeleteResponse(id=job.id, deleted=True)
@router.get("/v1/videos/{video_id}/content")
@router.get("/v1/videos/{video_id}/content", dependencies=[Depends(_require_single_api_video_store)])
async def download_video(video_id: str) -> Response:
"""Download the generated file for a completed video job.
@@ -2662,6 +2782,7 @@ async def stop_profile(raw_request: Request, request: ProfileRequest | None = No
@router.post("/v1/omni/sleep")
async def omni_sleep(request: OmniSleepRequest, raw_request: Request):
_reject_process_local_state_with_multiple_api_workers(raw_request, "Sleep")
engine_client = raw_request.app.state.engine_client
sleeping_set = raw_request.app.state.sleeping_stages
if not hasattr(engine_client, "sleep"):
@@ -2674,6 +2795,7 @@ async def omni_sleep(request: OmniSleepRequest, raw_request: Request):
@router.post("/v1/omni/wakeup")
async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request):
_reject_process_local_state_with_multiple_api_workers(raw_request, "Wakeup")
engine_client = raw_request.app.state.engine_client
sleeping_set = raw_request.app.state.sleeping_stages
if not any(sid in sleeping_set for sid in request.stage_ids):