Rebase multi-API serving on latest main
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
78934753ff
commit
dc3c7dbe16
25 files changed
+1832
-202
No files matched your search
@@ -6,6 +6,7 @@ This module owns app construction, server startup, app-state initialization,
|
||||
and route bodies that have not yet moved to endpoint-owned modules."""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import dataclasses
|
||||
import json
|
||||
import multiprocessing
|
||||
@@ -14,6 +15,8 @@ import os
|
||||
|
||||
# Image generation API imports
|
||||
import random
|
||||
import signal
|
||||
import socket
|
||||
import time
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncIterator
|
||||
@@ -21,6 +24,7 @@ from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import uvloop
|
||||
import vllm.envs as envs
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile, WebSocket
|
||||
from fastapi.responses import FileResponse, JSONResponse, Response, StreamingResponse
|
||||
@@ -80,12 +84,14 @@ from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import POOLING_TASKS
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.system_utils import decorate_logs
|
||||
from vllm.utils.system_utils import decorate_logs, set_process_title
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
from vllm_omni.config.endpoint_policy import (
|
||||
shutdown_unsupported_routes,
|
||||
)
|
||||
from vllm_omni.engine.stage_init_utils import set_death_signal
|
||||
from vllm_omni.engine.stage_runtime import OmniClientConfig
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.duplex.capability import should_enable_duplex_endpoint
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
@@ -229,8 +235,55 @@ async def omni_run_server(args, **uvicorn_kwargs) -> None:
|
||||
await omni_run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||
|
||||
|
||||
async def omni_run_server_worker(listen_address, sock, args, client_config=None, **uvicorn_kwargs) -> None:
|
||||
def run_omni_api_server_worker_proc(
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: TrackingNamespace,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
**uvicorn_kwargs: object,
|
||||
) -> None:
|
||||
"""Entrypoint used by vLLM's API server process manager."""
|
||||
set_death_signal(signal.SIGTERM)
|
||||
manager_config = client_config or {}
|
||||
client_index = int(manager_config.get("client_index", 0))
|
||||
all_client_configs = getattr(args, "_omni_stage_client_configs", None)
|
||||
if not all_client_configs or not 0 <= client_index < len(all_client_configs):
|
||||
raise RuntimeError(f"Missing Omni stage client configuration for API server {client_index}")
|
||||
|
||||
omni_client_config: OmniClientConfig = copy.deepcopy(all_client_configs[client_index])
|
||||
omni_client_config["client_count"] = int(manager_config.get("client_count", 1))
|
||||
omni_client_config["client_index"] = client_index
|
||||
|
||||
stage_addresses = omni_client_config["stage_addresses"]
|
||||
first_stage_id = min(stage_addresses)
|
||||
first_replica_id = min(stage_addresses[first_stage_id])
|
||||
first_addresses = stage_addresses[first_stage_id][first_replica_id]
|
||||
for key in ("input_address", "output_address", "actual_address_pipe", "tensor_queue"):
|
||||
if key in manager_config:
|
||||
first_addresses[key] = manager_config[key]
|
||||
|
||||
set_process_title("APIServer", str(client_index))
|
||||
decorate_logs("APIServer", skip_if_decorated=True)
|
||||
uvloop.run(
|
||||
omni_run_server_worker(
|
||||
listen_address,
|
||||
sock,
|
||||
args,
|
||||
client_config=omni_client_config,
|
||||
**uvicorn_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def omni_run_server_worker(
|
||||
listen_address: str,
|
||||
sock: socket.socket,
|
||||
args: TrackingNamespace,
|
||||
client_config: OmniClientConfig | None = None,
|
||||
**uvicorn_kwargs: object,
|
||||
) -> None:
|
||||
"""Run a single API server worker."""
|
||||
api_server_count = _resolve_api_server_count(args, client_config)
|
||||
|
||||
if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
|
||||
ToolParserManager.import_tool_parser(args.tool_parser_plugin)
|
||||
@@ -270,6 +323,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
|
||||
# OMNI: Pass supported_tasks to build_app (required by upstream vLLM)
|
||||
app = build_openai_app(args, supported_tasks)
|
||||
app.state.api_server_count = api_server_count
|
||||
|
||||
# OMNI: Remove upstream routes that we override with omni-specific handlers
|
||||
remove_route_from_app(app, "/v1/chat/completions", {"POST"})
|
||||
@@ -345,6 +399,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
if scope["type"] == "http":
|
||||
scope.setdefault("state", {})
|
||||
scope["state"]["request_timestamp"] = time.time()
|
||||
|
||||
await self._inner(scope, receive, send)
|
||||
|
||||
# Startup duplex warmup (duplex_session.warmup_frames in the deploy
|
||||
@@ -403,7 +458,7 @@ async def build_async_omni(
|
||||
args: TrackingNamespace,
|
||||
*,
|
||||
disable_frontend_multiprocessing: bool | None = None,
|
||||
client_config: dict[str, Any] | None = None,
|
||||
client_config: OmniClientConfig | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
"""Build an AsyncOmni instance from command-line arguments.
|
||||
|
||||
@@ -434,6 +489,7 @@ async def build_async_omni(
|
||||
async with build_async_omni_from_stage_config(
|
||||
args,
|
||||
disable_frontend_multiprocessing=disable_frontend_multiprocessing,
|
||||
client_config=client_config,
|
||||
) as async_omni:
|
||||
yield async_omni
|
||||
|
||||
@@ -443,6 +499,7 @@ async def build_async_omni_from_stage_config(
|
||||
args: TrackingNamespace,
|
||||
*,
|
||||
disable_frontend_multiprocessing: bool = False,
|
||||
client_config: OmniClientConfig | None = None,
|
||||
) -> AsyncIterator[EngineClient]:
|
||||
"""Create AsyncOmni from stage configuration.
|
||||
|
||||
@@ -499,6 +556,8 @@ async def build_async_omni_from_stage_config(
|
||||
kwargs.pop("robot_openpi_idle_timeout", None)
|
||||
model = kwargs.pop("model", None) or args.model
|
||||
kwargs.setdefault("log_stats", not args.disable_log_stats)
|
||||
if client_config is not None:
|
||||
kwargs["client_config"] = client_config
|
||||
async_omni = AsyncOmni(model=model, **kwargs)
|
||||
|
||||
# # Don't keep the dummy data in memory
|
||||
@@ -1272,6 +1331,42 @@ async def list_voices(raw_request: Request):
|
||||
return JSONResponse(content={"voices": speakers, "uploaded_voices": uploaded_speakers})
|
||||
|
||||
|
||||
def _resolve_api_server_count(args: Namespace, client_config: OmniClientConfig | None) -> int:
|
||||
"""Resolve frontend topology once, from the launcher's client configuration."""
|
||||
requested = getattr(args, "api_server_count", None)
|
||||
actual = 1 if client_config is None else client_config.get("client_count")
|
||||
if type(actual) is not int or actual < 1:
|
||||
raise ValueError("API client configuration requires a positive integer client_count")
|
||||
if requested is not None and (type(requested) is not int or requested < 1 or requested != actual):
|
||||
raise ValueError("API server count does not match the frontend launch configuration")
|
||||
return actual
|
||||
|
||||
|
||||
def _reject_process_local_state_with_multiple_api_workers(raw_request: Request, operation: str) -> None:
|
||||
"""Reject access to frontend state that is not shared across API workers."""
|
||||
api_server_count = getattr(raw_request.app.state, "api_server_count", None)
|
||||
if type(api_server_count) is not int or api_server_count < 1:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
detail="API worker topology is not initialized.",
|
||||
)
|
||||
if api_server_count > 1:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.CONFLICT.value,
|
||||
detail=(
|
||||
f"{operation} is not supported with --api-server-count > 1 because "
|
||||
"the operation uses process-local frontend state"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _require_single_api_video_store(raw_request: Request) -> None:
|
||||
# VIDEO_STORE and VIDEO_TASKS are process-local, including reads and
|
||||
# cancellation. Keep this independent of the temporary diffusion launch
|
||||
# restriction: enabling multi-API diffusion alone cannot make jobs shared.
|
||||
_reject_process_local_state_with_multiple_api_workers(raw_request, "Asynchronous video jobs")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/audio/voices",
|
||||
responses={
|
||||
@@ -1317,6 +1412,7 @@ async def upload_voice(
|
||||
Returns:
|
||||
JSON response with voice information
|
||||
"""
|
||||
_reject_process_local_state_with_multiple_api_workers(raw_request, "Runtime voice upload")
|
||||
handler = Omnispeech(raw_request)
|
||||
if handler is None:
|
||||
return _create_speech_error_json_response(
|
||||
@@ -1382,6 +1478,7 @@ async def delete_voice(name: str, raw_request: Request):
|
||||
Returns:
|
||||
JSON response indicating success or failure
|
||||
"""
|
||||
_reject_process_local_state_with_multiple_api_workers(raw_request, "Runtime voice deletion")
|
||||
handler = Omnispeech(raw_request)
|
||||
if handler is None:
|
||||
return _create_speech_error_json_response(
|
||||
@@ -1489,6 +1586,8 @@ async def realtime_websocket(websocket: WebSocket):
|
||||
duplex_query is None or (isinstance(duplex_query, str) and duplex_query.lower() in {"1", "true", "on"})
|
||||
)
|
||||
if use_duplex_realtime and duplex_handler is not None:
|
||||
if await _reject_multi_api_duplex(websocket):
|
||||
return
|
||||
await duplex_handler.handle_realtime_session(websocket)
|
||||
return
|
||||
|
||||
@@ -1535,6 +1634,8 @@ async def realtime_robot_openpi(websocket: WebSocket):
|
||||
@router.websocket("/v1/duplex")
|
||||
async def duplex_websocket(websocket: WebSocket):
|
||||
"""WebSocket endpoint for vLLM-Omni duplex session control."""
|
||||
if await _reject_multi_api_duplex(websocket):
|
||||
return
|
||||
handler = getattr(websocket.app.state, "openai_serving_duplex", None)
|
||||
if handler is None:
|
||||
await websocket.accept()
|
||||
@@ -1544,6 +1645,24 @@ async def duplex_websocket(websocket: WebSocket):
|
||||
await handler.handle_session(websocket)
|
||||
|
||||
|
||||
async def _reject_multi_api_duplex(websocket: WebSocket) -> bool:
|
||||
# Session credentials and replay state are local to each frontend; a shared
|
||||
# listening socket cannot route reconnects back to the session owner.
|
||||
count = getattr(websocket.app.state, "api_server_count", None)
|
||||
if count is None or (type(count) is int and count == 1):
|
||||
return False
|
||||
await websocket.accept()
|
||||
await websocket.send_json(
|
||||
{
|
||||
"type": "error",
|
||||
"error": "Duplex sessions require a single API worker with initialized topology.",
|
||||
"code": "multi_api_duplex_unsupported",
|
||||
}
|
||||
)
|
||||
await websocket.close(code=1008)
|
||||
return True
|
||||
|
||||
|
||||
# RL Rollout serving (RFC #3747, P0)
|
||||
|
||||
|
||||
@@ -2311,6 +2430,7 @@ async def edit_images(
|
||||
|
||||
@router.post(
|
||||
"/v1/videos",
|
||||
dependencies=[Depends(_require_single_api_video_store)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"model": VideoResponse},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
@@ -2452,7 +2572,7 @@ async def create_video_sync(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/v1/videos", response_model=VideoListResponse)
|
||||
@router.get("/v1/videos", response_model=VideoListResponse, dependencies=[Depends(_require_single_api_video_store)])
|
||||
async def list_videos(
|
||||
after: str | None = None,
|
||||
limit: int | None = Query(None, ge=0, le=100),
|
||||
@@ -2489,7 +2609,7 @@ async def list_videos(
|
||||
return VideoListResponse(data=jobs, has_more=has_more, first_id=first_id, last_id=last_id)
|
||||
|
||||
|
||||
@router.get("/v1/videos/{video_id}", response_model=None)
|
||||
@router.get("/v1/videos/{video_id}", response_model=None, dependencies=[Depends(_require_single_api_video_store)])
|
||||
async def retrieve_video(video_id: str) -> VideoResponse | JSONResponse:
|
||||
"""Retrieve metadata for a previously created video job.
|
||||
|
||||
@@ -2517,7 +2637,7 @@ async def retrieve_video(video_id: str) -> VideoResponse | JSONResponse:
|
||||
return job
|
||||
|
||||
|
||||
@router.delete("/v1/videos/{video_id}")
|
||||
@router.delete("/v1/videos/{video_id}", dependencies=[Depends(_require_single_api_video_store)])
|
||||
async def delete_video(video_id: str) -> VideoDeleteResponse:
|
||||
"""Delete a stored video job and any generated output.
|
||||
|
||||
@@ -2569,7 +2689,7 @@ async def delete_video(video_id: str) -> VideoDeleteResponse:
|
||||
return VideoDeleteResponse(id=job.id, deleted=True)
|
||||
|
||||
|
||||
@router.get("/v1/videos/{video_id}/content")
|
||||
@router.get("/v1/videos/{video_id}/content", dependencies=[Depends(_require_single_api_video_store)])
|
||||
async def download_video(video_id: str) -> Response:
|
||||
"""Download the generated file for a completed video job.
|
||||
|
||||
@@ -2662,6 +2782,7 @@ async def stop_profile(raw_request: Request, request: ProfileRequest | None = No
|
||||
|
||||
@router.post("/v1/omni/sleep")
|
||||
async def omni_sleep(request: OmniSleepRequest, raw_request: Request):
|
||||
_reject_process_local_state_with_multiple_api_workers(raw_request, "Sleep")
|
||||
engine_client = raw_request.app.state.engine_client
|
||||
sleeping_set = raw_request.app.state.sleeping_stages
|
||||
if not hasattr(engine_client, "sleep"):
|
||||
@@ -2674,6 +2795,7 @@ async def omni_sleep(request: OmniSleepRequest, raw_request: Request):
|
||||
|
||||
@router.post("/v1/omni/wakeup")
|
||||
async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request):
|
||||
_reject_process_local_state_with_multiple_api_workers(raw_request, "Wakeup")
|
||||
engine_client = raw_request.app.state.engine_client
|
||||
sleeping_set = raw_request.app.state.sleeping_stages
|
||||
if not any(sid in sleeping_set for sid in request.stage_ids):
|
||||
|
||||
Reference in new issue
Block a user