Merge main and preserve multi-API engine and duplex guards
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
commit
daf91b4d7a
329 files changed
+36428
-35780
No files matched your search
@@ -92,10 +92,10 @@ from vllm_omni.config.endpoint_policy import (
|
||||
)
|
||||
from vllm_omni.engine.stage_init_utils import set_death_signal
|
||||
from vllm_omni.engine.stage_runtime import OmniClientConfig
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.duplex.capability import should_enable_duplex_endpoint
|
||||
from vllm_omni.entrypoints.async_omni import ABORT_TIMEOUT_S, AsyncOmni
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
from vllm_omni.entrypoints.duplex.warmup import _warmup_duplex_realtime
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
from vllm_omni.entrypoints.openai import app_state as openai_app_state
|
||||
from vllm_omni.entrypoints.openai.app_state import (
|
||||
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL,
|
||||
@@ -207,6 +207,8 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser, TrackingName
|
||||
logger = init_logger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
VIDEO_ABORT_TIMEOUT_S = ABORT_TIMEOUT_S
|
||||
|
||||
profiler_router = APIRouter()
|
||||
|
||||
|
||||
@@ -494,6 +496,23 @@ async def build_async_omni(
|
||||
yield async_omni
|
||||
|
||||
|
||||
def _is_duplex_model(model: str, kwargs: dict[str, Any]) -> bool:
|
||||
"""Whether ``model``'s pipeline declares a ``duplex_plugin`` (served by ``DuplexOmni``).
|
||||
|
||||
Resolution errors propagate: a duplex model whose pipeline or deploy
|
||||
config cannot be resolved must fail startup rather than silently start a
|
||||
turn-based server.
|
||||
"""
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
|
||||
pipeline_config = StageConfigFactory.get_pipeline_config(
|
||||
model=model,
|
||||
trust_remote_code=bool(kwargs.get("trust_remote_code")),
|
||||
deploy_config_path=kwargs.get("deploy_config"),
|
||||
)
|
||||
return bool(pipeline_config is not None and getattr(pipeline_config, "duplex_plugin", None))
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_omni_from_stage_config(
|
||||
args: TrackingNamespace,
|
||||
@@ -558,7 +577,12 @@ async def build_async_omni_from_stage_config(
|
||||
kwargs.setdefault("log_stats", not args.disable_log_stats)
|
||||
if client_config is not None:
|
||||
kwargs["client_config"] = client_config
|
||||
async_omni = AsyncOmni(model=model, **kwargs)
|
||||
if _is_duplex_model(model, kwargs):
|
||||
# A duplex model is always served in duplex mode: sessions over
|
||||
# /v1/realtime?duplex=1, no turn-based HTTP routes.
|
||||
async_omni = DuplexOmni(model=model, **kwargs)
|
||||
else:
|
||||
async_omni = AsyncOmni(model=model, **kwargs)
|
||||
|
||||
# # Don't keep the dummy data in memory
|
||||
# await async_llm.reset_mm_cache()
|
||||
@@ -569,6 +593,134 @@ async def build_async_omni_from_stage_config(
|
||||
async_omni.shutdown()
|
||||
|
||||
|
||||
async def _init_duplex_app_state(
|
||||
engine_client: DuplexOmni,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
base_model_paths: list[BaseModelPath],
|
||||
vllm_config: Any,
|
||||
request_logger: RequestLogger | None,
|
||||
) -> None:
|
||||
"""Minimal app state for a duplex server: only the surfaces a duplex session backs."""
|
||||
state.vllm_config = vllm_config
|
||||
state.diffusion_engine = None
|
||||
state.openai_serving_models = OpenAIServingModels(
|
||||
engine_client=engine_client, # type: ignore[arg-type]
|
||||
base_model_paths=base_model_paths,
|
||||
lora_modules=None,
|
||||
)
|
||||
state.serving_tokenization = None
|
||||
state.serving_tokens = None
|
||||
# Replaced by the chat init below when the model serves chat.
|
||||
state.online_renderer = None
|
||||
for attribute in (
|
||||
"openai_serving_chat_batch",
|
||||
"openai_serving_completion",
|
||||
"openai_serving_responses",
|
||||
"openai_serving_embedding",
|
||||
"openai_serving_pooling",
|
||||
"openai_serving_classification",
|
||||
"openai_serving_scores",
|
||||
"openai_serving_transcription",
|
||||
"openai_serving_translation",
|
||||
"openai_serving_speech",
|
||||
"openai_serving_audio_generate",
|
||||
"openai_serving_video",
|
||||
"openai_streaming_speech",
|
||||
"openai_streaming_video",
|
||||
"openai_streaming_video_output",
|
||||
"openai_serving_realtime",
|
||||
"openai_serving_realtime_robot",
|
||||
"anthropic_serving_messages",
|
||||
):
|
||||
setattr(state, attribute, None)
|
||||
state.openai_serving_duplex = OmniDuplexSessionHandler(duplex_omni=engine_client)
|
||||
# One engine, both surfaces. ``DuplexOmni`` extends ``AsyncOmni``, so the
|
||||
# ordinary chat service runs on it unchanged: a chat request is a turn-based
|
||||
# request on the same pipeline, not a session, and costs no admission slot.
|
||||
state.openai_serving_chat = await _init_duplex_chat(engine_client, state, args, request_logger)
|
||||
state.enable_server_load_tracking = getattr(args, "enable_server_load_tracking", False)
|
||||
state.server_load_metrics = 0
|
||||
if state.openai_serving_chat is not None:
|
||||
logger.info(
|
||||
"Duplex mode: serving %s over /v1/realtime?duplex=1 and /v1/chat/completions",
|
||||
engine_client.model,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Duplex mode: serving %s over /v1/realtime?duplex=1 only "
|
||||
"(/v1/chat/completions unavailable: the model does not declare supports_chat_completions)",
|
||||
engine_client.model,
|
||||
)
|
||||
|
||||
|
||||
async def _init_duplex_chat(
|
||||
engine_client: DuplexOmni,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
) -> OmniOpenAIServingChat | None:
|
||||
"""The ordinary chat service, on a duplex engine, when the model allows it.
|
||||
|
||||
Gated on ``DuplexCapabilities.supports_chat_completions`` so the decision
|
||||
stays the model's: a duplex model that should not answer chat requests says
|
||||
so in its plugin, and the route reports "not available" rather than
|
||||
answering badly. ``endpoint_restrictions`` remains the per-deployment
|
||||
opt-out on top of this.
|
||||
"""
|
||||
if not engine_client.duplex_capabilities.supports_chat_completions:
|
||||
return None
|
||||
supported_tasks: set[str] = {"generate"}
|
||||
if hasattr(engine_client, "get_supported_tasks"):
|
||||
supported_tasks = set(await engine_client.get_supported_tasks())
|
||||
if "generate" not in supported_tasks:
|
||||
return None
|
||||
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
if resolved_chat_template is None:
|
||||
try:
|
||||
tokenizer = await engine_client.get_tokenizer()
|
||||
except Exception as exc:
|
||||
logger.debug("Could not inspect tokenizer chat_template before duplex chat init: %s", exc)
|
||||
tokenizer = None
|
||||
if tokenizer is None or getattr(tokenizer, "chat_template", None) is None:
|
||||
resolved_chat_template = _load_model_chat_template_json(args.model)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=engine_client.model_config,
|
||||
renderer=engine_client.renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
)
|
||||
return OmniOpenAIServingChat(
|
||||
engine_client=engine_client,
|
||||
models=state.openai_serving_models,
|
||||
response_role=args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
)
|
||||
|
||||
|
||||
async def omni_init_app_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
@@ -619,6 +771,13 @@ async def omni_init_app_state(
|
||||
state.stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None
|
||||
model_name = served_model_names[0] if served_model_names else args.model
|
||||
|
||||
# Duplex mode: a duplex model is served through DuplexOmni only. Sessions
|
||||
# run over /v1/realtime?duplex=1; every turn-based HTTP route reports
|
||||
# "not available".
|
||||
if isinstance(engine_client, DuplexOmni):
|
||||
await _init_duplex_app_state(engine_client, state, args, base_model_paths, vllm_config, request_logger)
|
||||
return
|
||||
|
||||
# Pure Diffusion mode: use simplified initialization logic
|
||||
if is_pure_diffusion:
|
||||
state.vllm_config = None
|
||||
@@ -999,17 +1158,6 @@ async def omni_init_app_state(
|
||||
else None
|
||||
)
|
||||
state.openai_serving_duplex = None
|
||||
if state.openai_serving_chat is not None and should_enable_duplex_endpoint(
|
||||
state.stage_configs,
|
||||
config_path=getattr(engine_client, "config_path", None) or getattr(args, "deploy_config", None),
|
||||
):
|
||||
state.openai_serving_duplex = OmniDuplexSessionHandler(
|
||||
chat_service=state.openai_serving_chat,
|
||||
served_model_name=model_name,
|
||||
log_stats=state.log_stats,
|
||||
duplex_session_config=getattr(engine_client, "duplex_session_config", None),
|
||||
serving_runtime_adapter_path=getattr(engine_client, "duplex_serving_adapter_path", None),
|
||||
)
|
||||
state.openai_serving_realtime = OpenAIServingRealtime(
|
||||
engine_client=engine_client,
|
||||
models=state.openai_serving_models,
|
||||
@@ -1572,14 +1720,6 @@ async def streaming_video_output(websocket: WebSocket):
|
||||
@router.websocket("/v1/realtime")
|
||||
async def realtime_websocket(websocket: WebSocket):
|
||||
"""WebSocket endpoint for OpenAI-style realtime interactions."""
|
||||
# Hold real clients until the startup duplex warmup finishes (the warmup
|
||||
# connection marks itself with vllm_omni_warmup=1 and passes through).
|
||||
warmup_done = getattr(websocket.app.state, "duplex_warmup_done", None)
|
||||
if warmup_done is not None and not warmup_done.is_set() and websocket.query_params.get("vllm_omni_warmup") != "1":
|
||||
try:
|
||||
await asyncio.wait_for(warmup_done.wait(), timeout=120)
|
||||
except (TimeoutError, asyncio.TimeoutError):
|
||||
logger.warning("Duplex warmup still running after 120 s; admitting the client anyway.")
|
||||
duplex_handler = getattr(websocket.app.state, "openai_serving_duplex", None)
|
||||
duplex_query = websocket.query_params.get("duplex")
|
||||
use_duplex_realtime = duplex_handler is not None and (
|
||||
@@ -1588,6 +1728,7 @@ async def realtime_websocket(websocket: WebSocket):
|
||||
if use_duplex_realtime and duplex_handler is not None:
|
||||
if await _reject_multi_api_duplex(websocket):
|
||||
return
|
||||
await _wait_for_duplex_warmup(websocket)
|
||||
await duplex_handler.handle_realtime_session(websocket)
|
||||
return
|
||||
|
||||
@@ -1601,6 +1742,34 @@ async def realtime_websocket(websocket: WebSocket):
|
||||
await connection.handle_connection()
|
||||
|
||||
|
||||
async def _wait_for_duplex_warmup(websocket: WebSocket) -> None:
|
||||
"""Hold real clients until the startup duplex warmup finishes.
|
||||
|
||||
The warmup connection marks itself with ``vllm_omni_warmup=1`` and passes through.
|
||||
"""
|
||||
warmup_done = getattr(websocket.app.state, "duplex_warmup_done", None)
|
||||
if warmup_done is not None and not warmup_done.is_set() and websocket.query_params.get("vllm_omni_warmup") != "1":
|
||||
try:
|
||||
await asyncio.wait_for(warmup_done.wait(), timeout=120)
|
||||
except (TimeoutError, asyncio.TimeoutError):
|
||||
logger.warning("Duplex warmup still running after 120 s; admitting the client anyway.")
|
||||
|
||||
|
||||
@router.websocket("/v1/duplex")
|
||||
async def duplex_websocket(websocket: WebSocket):
|
||||
"""Alias of ``/v1/realtime?duplex=1``: the same Realtime duplex session protocol."""
|
||||
if await _reject_multi_api_duplex(websocket):
|
||||
return
|
||||
await _wait_for_duplex_warmup(websocket)
|
||||
handler = getattr(websocket.app.state, "openai_serving_duplex", None)
|
||||
if handler is None:
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "error": "Duplex API is not available", "code": "unsupported"})
|
||||
await websocket.close()
|
||||
return
|
||||
await handler.handle_realtime_session(websocket)
|
||||
|
||||
|
||||
@router.websocket("/v1/realtime/robot/openpi")
|
||||
async def realtime_robot_openpi(websocket: WebSocket):
|
||||
"""WebSocket endpoint for robot policy inference via OpenPI messages."""
|
||||
@@ -1631,20 +1800,6 @@ async def realtime_robot_openpi(websocket: WebSocket):
|
||||
await connection.handle_connection()
|
||||
|
||||
|
||||
@router.websocket("/v1/duplex")
|
||||
async def duplex_websocket(websocket: WebSocket):
|
||||
"""WebSocket endpoint for vLLM-Omni duplex session control."""
|
||||
if await _reject_multi_api_duplex(websocket):
|
||||
return
|
||||
handler = getattr(websocket.app.state, "openai_serving_duplex", None)
|
||||
if handler is None:
|
||||
await websocket.accept()
|
||||
await websocket.send_json({"type": "error", "error": "Duplex API is not available", "code": "unsupported"})
|
||||
await websocket.close()
|
||||
return
|
||||
await handler.handle_session(websocket)
|
||||
|
||||
|
||||
async def _reject_multi_api_duplex(websocket: WebSocket) -> bool:
|
||||
# Session credentials and replay state are local to each frontend; a shared
|
||||
# listening socket cannot route reconnects back to the session owner.
|
||||
@@ -2638,11 +2793,11 @@ async def retrieve_video(video_id: str) -> VideoResponse | JSONResponse:
|
||||
|
||||
|
||||
@router.delete("/v1/videos/{video_id}", dependencies=[Depends(_require_single_api_video_store)])
|
||||
async def delete_video(video_id: str) -> VideoDeleteResponse:
|
||||
async def delete_video(video_id: str, raw_request: Request) -> VideoDeleteResponse:
|
||||
"""Delete a stored video job and any generated output.
|
||||
|
||||
If the job is still queued or running, this endpoint first attempts to
|
||||
cancel the in-flight generation task before removing the stored metadata.
|
||||
In-flight jobs get a bounded engine abort, then frontend cancel. The job
|
||||
is re-read afterwards so a completed save is not orphaned.
|
||||
|
||||
Args:
|
||||
video_id: Identifier of the video job to delete.
|
||||
@@ -2659,19 +2814,37 @@ async def delete_video(video_id: str) -> VideoDeleteResponse:
|
||||
raise HTTPException(status_code=404, detail="Video not found")
|
||||
|
||||
if job.status in (VideoGenerationStatus.QUEUED, VideoGenerationStatus.IN_PROGRESS):
|
||||
handler = raw_request.app.state.openai_serving_video
|
||||
try:
|
||||
await asyncio.wait_for(handler.abort_request(video_id), timeout=VIDEO_ABORT_TIMEOUT_S)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"Timed out aborting video request %s after %.1fs; "
|
||||
"engine abort is best-effort until the current batch drains",
|
||||
video_id,
|
||||
VIDEO_ABORT_TIMEOUT_S,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to abort in-flight video request %s", video_id)
|
||||
task = await VIDEO_TASKS.get(video_id)
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
# Cancel cleanup may spend a full abort budget; +2s covers scheduling slack.
|
||||
await asyncio.wait_for(task, timeout=VIDEO_ABORT_TIMEOUT_S + 2.0)
|
||||
except asyncio.TimeoutError:
|
||||
raise HTTPException(status_code=409, detail="Cancellation in progress. Please try again later.")
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
job = await VIDEO_STORE.get(video_id)
|
||||
if job is None:
|
||||
return VideoDeleteResponse(id=video_id, deleted=True)
|
||||
if job.status in (VideoGenerationStatus.QUEUED, VideoGenerationStatus.IN_PROGRESS):
|
||||
await VIDEO_STORE.pop(video_id)
|
||||
return VideoDeleteResponse(id=job.id, deleted=True)
|
||||
elif job.status is VideoGenerationStatus.FAILED:
|
||||
|
||||
if job.status is VideoGenerationStatus.FAILED:
|
||||
if job.file_name is not None:
|
||||
try:
|
||||
await STORAGE_MANAGER.delete(video_id)
|
||||
|
||||
Reference in new issue
Block a user