[Core][Frontend] Unified Full-duplex Framework (#7413)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
chickeytonandClaude Opus 5 authored and GitHub committed 2026-09-16 15:35:36 +08:00
1 parent 9005d78903
commit 99ff4f3070
182 files changed
+26951 -34743

No files matched your search

+186 -33
View File
@@ -87,9 +87,9 @@ from vllm_omni.config.endpoint_policy import (
shutdown_unsupported_routes,
)
from vllm_omni.entrypoints.async_omni import ABORT_TIMEOUT_S, AsyncOmni
from vllm_omni.entrypoints.duplex.capability import should_enable_duplex_endpoint
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
from vllm_omni.entrypoints.duplex.warmup import _warmup_duplex_realtime
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
from vllm_omni.entrypoints.openai import app_state as openai_app_state
from vllm_omni.entrypoints.openai.app_state import (
ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL,
@@ -440,6 +440,23 @@ async def build_async_omni(
yield async_omni
def _is_duplex_model(model: str, kwargs: dict[str, Any]) -> bool:
"""Whether ``model``'s pipeline declares a ``duplex_plugin`` (served by ``DuplexOmni``).
Resolution errors propagate: a duplex model whose pipeline or deploy
config cannot be resolved must fail startup rather than silently start a
turn-based server.
"""
from vllm_omni.config.config_factory import StageConfigFactory
pipeline_config = StageConfigFactory.get_pipeline_config(
model=model,
trust_remote_code=bool(kwargs.get("trust_remote_code")),
deploy_config_path=kwargs.get("deploy_config"),
)
return bool(pipeline_config is not None and getattr(pipeline_config, "duplex_plugin", None))
@asynccontextmanager
async def build_async_omni_from_stage_config(
args: TrackingNamespace,
@@ -501,7 +518,12 @@ async def build_async_omni_from_stage_config(
kwargs.pop("robot_openpi_idle_timeout", None)
model = kwargs.pop("model", None) or args.model
kwargs.setdefault("log_stats", not args.disable_log_stats)
async_omni = AsyncOmni(model=model, **kwargs)
if _is_duplex_model(model, kwargs):
# A duplex model is always served in duplex mode: sessions over
# /v1/realtime?duplex=1, no turn-based HTTP routes.
async_omni = DuplexOmni(model=model, **kwargs)
else:
async_omni = AsyncOmni(model=model, **kwargs)
# # Don't keep the dummy data in memory
# await async_llm.reset_mm_cache()
@@ -512,6 +534,134 @@ async def build_async_omni_from_stage_config(
async_omni.shutdown()
async def _init_duplex_app_state(
engine_client: DuplexOmni,
state: State,
args: Namespace,
base_model_paths: list[BaseModelPath],
vllm_config: Any,
request_logger: RequestLogger | None,
) -> None:
"""Minimal app state for a duplex server: only the surfaces a duplex session backs."""
state.vllm_config = vllm_config
state.diffusion_engine = None
state.openai_serving_models = OpenAIServingModels(
engine_client=engine_client, # type: ignore[arg-type]
base_model_paths=base_model_paths,
lora_modules=None,
)
state.serving_tokenization = None
state.serving_tokens = None
# Replaced by the chat init below when the model serves chat.
state.online_renderer = None
for attribute in (
"openai_serving_chat_batch",
"openai_serving_completion",
"openai_serving_responses",
"openai_serving_embedding",
"openai_serving_pooling",
"openai_serving_classification",
"openai_serving_scores",
"openai_serving_transcription",
"openai_serving_translation",
"openai_serving_speech",
"openai_serving_audio_generate",
"openai_serving_video",
"openai_streaming_speech",
"openai_streaming_video",
"openai_streaming_video_output",
"openai_serving_realtime",
"openai_serving_realtime_robot",
"anthropic_serving_messages",
):
setattr(state, attribute, None)
state.openai_serving_duplex = OmniDuplexSessionHandler(duplex_omni=engine_client)
# One engine, both surfaces. ``DuplexOmni`` extends ``AsyncOmni``, so the
# ordinary chat service runs on it unchanged: a chat request is a turn-based
# request on the same pipeline, not a session, and costs no admission slot.
state.openai_serving_chat = await _init_duplex_chat(engine_client, state, args, request_logger)
state.enable_server_load_tracking = getattr(args, "enable_server_load_tracking", False)
state.server_load_metrics = 0
if state.openai_serving_chat is not None:
logger.info(
"Duplex mode: serving %s over /v1/realtime?duplex=1 and /v1/chat/completions",
engine_client.model,
)
else:
logger.info(
"Duplex mode: serving %s over /v1/realtime?duplex=1 only "
"(/v1/chat/completions unavailable: the model does not declare supports_chat_completions)",
engine_client.model,
)
async def _init_duplex_chat(
engine_client: DuplexOmni,
state: State,
args: Namespace,
request_logger: RequestLogger | None,
) -> OmniOpenAIServingChat | None:
"""The ordinary chat service, on a duplex engine, when the model allows it.
Gated on ``DuplexCapabilities.supports_chat_completions`` so the decision
stays the model's: a duplex model that should not answer chat requests says
so in its plugin, and the route reports "not available" rather than
answering badly. ``endpoint_restrictions`` remains the per-deployment
opt-out on top of this.
"""
if not engine_client.duplex_capabilities.supports_chat_completions:
return None
supported_tasks: set[str] = {"generate"}
if hasattr(engine_client, "get_supported_tasks"):
supported_tasks = set(await engine_client.get_supported_tasks())
if "generate" not in supported_tasks:
return None
resolved_chat_template = load_chat_template(args.chat_template)
if resolved_chat_template is None:
try:
tokenizer = await engine_client.get_tokenizer()
except Exception as exc:
logger.debug("Could not inspect tokenizer chat_template before duplex chat init: %s", exc)
tokenizer = None
if tokenizer is None or getattr(tokenizer, "chat_template", None) is None:
resolved_chat_template = _load_model_chat_template_json(args.model)
state.online_renderer = OnlineRenderer(
model_config=engine_client.model_config,
renderer=engine_client.renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
)
return OmniOpenAIServingChat(
engine_client=engine_client,
models=state.openai_serving_models,
response_role=args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
)
async def omni_init_app_state(
engine_client: EngineClient,
state: State,
@@ -562,6 +712,13 @@ async def omni_init_app_state(
state.stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None
model_name = served_model_names[0] if served_model_names else args.model
# Duplex mode: a duplex model is served through DuplexOmni only. Sessions
# run over /v1/realtime?duplex=1; every turn-based HTTP route reports
# "not available".
if isinstance(engine_client, DuplexOmni):
await _init_duplex_app_state(engine_client, state, args, base_model_paths, vllm_config, request_logger)
return
# Pure Diffusion mode: use simplified initialization logic
if is_pure_diffusion:
state.vllm_config = None
@@ -942,17 +1099,6 @@ async def omni_init_app_state(
else None
)
state.openai_serving_duplex = None
if state.openai_serving_chat is not None and should_enable_duplex_endpoint(
state.stage_configs,
config_path=getattr(engine_client, "config_path", None) or getattr(args, "deploy_config", None),
):
state.openai_serving_duplex = OmniDuplexSessionHandler(
chat_service=state.openai_serving_chat,
served_model_name=model_name,
log_stats=state.log_stats,
duplex_session_config=getattr(engine_client, "duplex_session_config", None),
serving_runtime_adapter_path=getattr(engine_client, "duplex_serving_adapter_path", None),
)
state.openai_serving_realtime = OpenAIServingRealtime(
engine_client=engine_client,
models=state.openai_serving_models,
@@ -1477,14 +1623,7 @@ async def streaming_video_output(websocket: WebSocket):
@router.websocket("/v1/realtime")
async def realtime_websocket(websocket: WebSocket):
"""WebSocket endpoint for OpenAI-style realtime interactions."""
# Hold real clients until the startup duplex warmup finishes (the warmup
# connection marks itself with vllm_omni_warmup=1 and passes through).
warmup_done = getattr(websocket.app.state, "duplex_warmup_done", None)
if warmup_done is not None and not warmup_done.is_set() and websocket.query_params.get("vllm_omni_warmup") != "1":
try:
await asyncio.wait_for(warmup_done.wait(), timeout=120)
except (TimeoutError, asyncio.TimeoutError):
logger.warning("Duplex warmup still running after 120 s; admitting the client anyway.")
await _wait_for_duplex_warmup(websocket)
duplex_handler = getattr(websocket.app.state, "openai_serving_duplex", None)
duplex_query = websocket.query_params.get("duplex")
use_duplex_realtime = duplex_handler is not None and (
@@ -1504,6 +1643,32 @@ async def realtime_websocket(websocket: WebSocket):
await connection.handle_connection()
async def _wait_for_duplex_warmup(websocket: WebSocket) -> None:
"""Hold real clients until the startup duplex warmup finishes.
The warmup connection marks itself with ``vllm_omni_warmup=1`` and passes through.
"""
warmup_done = getattr(websocket.app.state, "duplex_warmup_done", None)
if warmup_done is not None and not warmup_done.is_set() and websocket.query_params.get("vllm_omni_warmup") != "1":
try:
await asyncio.wait_for(warmup_done.wait(), timeout=120)
except (TimeoutError, asyncio.TimeoutError):
logger.warning("Duplex warmup still running after 120 s; admitting the client anyway.")
@router.websocket("/v1/duplex")
async def duplex_websocket(websocket: WebSocket):
"""Alias of ``/v1/realtime?duplex=1``: the same Realtime duplex session protocol."""
await _wait_for_duplex_warmup(websocket)
handler = getattr(websocket.app.state, "openai_serving_duplex", None)
if handler is None:
await websocket.accept()
await websocket.send_json({"type": "error", "error": "Duplex API is not available", "code": "unsupported"})
await websocket.close()
return
await handler.handle_realtime_session(websocket)
@router.websocket("/v1/realtime/robot/openpi")
async def realtime_robot_openpi(websocket: WebSocket):
"""WebSocket endpoint for robot policy inference via OpenPI messages."""
@@ -1534,18 +1699,6 @@ async def realtime_robot_openpi(websocket: WebSocket):
await connection.handle_connection()
@router.websocket("/v1/duplex")
async def duplex_websocket(websocket: WebSocket):
"""WebSocket endpoint for vLLM-Omni duplex session control."""
handler = getattr(websocket.app.state, "openai_serving_duplex", None)
if handler is None:
await websocket.accept()
await websocket.send_json({"type": "error", "error": "Duplex API is not available", "code": "unsupported"})
await websocket.close()
return
await handler.handle_session(websocket)
# RL Rollout serving (RFC #3747, P0)