[Frontend] Implement Batched Chat Completions (#5317)

Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
Alex Brooks authored and GitHub committed 2026-08-07 16:12:09 -06:00
1 parent 18bf7e0ff9
commit b15d4e6a64
7 files changed
+531 -39

No files matched your search

+76 -23
View File
@@ -39,6 +39,7 @@ from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, Tool
from vllm.entrypoints.openai.api_server import build_app as build_openai_app
from vllm.entrypoints.openai.api_server import setup_server as setup_openai_server
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
ChatCompletionResponse,
)
@@ -95,6 +96,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.config.endpoint_policy import shutdown_unsupported_routes
from vllm_omni.diffusion.models.interface import ReferenceVideoDecodeSpec
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.openai.batch_serving import OmniOpenAIServingChatBatch
from vllm_omni.entrypoints.openai.duplex_capability import should_enable_duplex_endpoint
from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError
from vllm_omni.entrypoints.openai.image_api_utils import (
@@ -513,6 +515,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
# OMNI: Remove upstream routes that we override with omni-specific handlers
_remove_route_from_app(app, "/v1/chat/completions", {"POST"})
_remove_route_from_app(app, "/v1/chat/completions/batch", {"POST"})
_remove_route_from_app(app, "/v1/models", {"GET"}) # Remove upstream /v1/models to use omni's handler
app.include_router(router)
@@ -777,6 +780,10 @@ async def omni_init_app_state(
diffusion_engine=engine_client, # type: ignore
model_name=model_name,
)
state.openai_serving_chat_batch = OmniOpenAIServingChatBatch.for_diffusion(
diffusion_engine=engine_client, # type: ignore
model_name=model_name,
)
# audio related
state.openai_serving_speech = None
@@ -950,30 +957,33 @@ async def omni_init_app_state(
if "generate" in supported_tasks
else None
)
state.openai_serving_chat = (
OmniOpenAIServingChat(
engine_client,
state.openai_serving_models,
args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
)
if "generate" in supported_tasks
else None
_chat_kwargs = dict(
engine_client=engine_client,
models=state.openai_serving_models,
response_role=args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
enable_log_deltas=args.enable_log_deltas,
)
state.openai_serving_chat = OmniOpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None
state.openai_serving_chat_batch = (
OmniOpenAIServingChatBatch(**_chat_kwargs) if "generate" in supported_tasks else None
)
# Warm up chat template processing to avoid first-request latency
if state.openai_serving_chat is not None:
state.openai_serving_chat.warmup()
@@ -1162,6 +1172,10 @@ def Omnichat(request: Request) -> OmniOpenAIServingChat | None:
return request.app.state.openai_serving_chat
def OmniBatchChat(request: Request) -> OmniOpenAIServingChatBatch | None:
return request.app.state.openai_serving_chat_batch
def Omnispeech(request: Request) -> OmniOpenAIServingSpeech | None:
return request.app.state.openai_serving_speech
@@ -1245,6 +1259,45 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
return StreamingResponse(content=generator, media_type="text/event-stream")
@router.post(
"/v1/chat/completions/batch",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_batch_chat_completion(request: BatchChatCompletionRequest, raw_request: Request):
handler = OmniBatchChat(raw_request)
if handler is None:
base_server = getattr(raw_request.app.state, "serving_tokenization", None)
if base_server is None:
raise HTTPException(
status_code=HTTPStatus.NOT_FOUND.value,
detail="The model does not support Chat Completions API",
)
return base_server.create_error_response(message="The model does not support Chat Completions API")
try:
result = await handler.create_batch_chat_completion(request, raw_request)
except (EngineGenerateError, EngineDeadError) as exc:
return _create_engine_error_json_response(raw_request, exc)
except Exception as e:
logger.exception("Batched chat completion failed: %s", e)
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
if isinstance(result, ErrorResponse):
return JSONResponse(
content=result.model_dump(),
status_code=result.error.code if result.error else 400,
)
return JSONResponse(content=result.model_dump(mode="json"))
_remove_route_from_router(router, "/v1/audio/speech", {"POST"})
@@ -0,0 +1,151 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import time
from fastapi import Request
from pydantic import ValidationError
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResponseChoice,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
UsageInfo,
)
from vllm.logger import init_logger
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
logger = init_logger(__name__)
class OmniOpenAIServingChatBatch(OmniOpenAIServingChat):
@staticmethod
def _maybe_collapse_choices(choices: list[ChatCompletionResponseChoice]):
"""Collapse content + audio choices into one choice.
NOTE: This is a hack for models that produce separate text and
audio choices to ensure we have 1:1 mapping between number of
inputs in the batch and responses.
"""
content_choice, audio_choice = None, None
if len(choices) == 1:
return choices[0]
if len(choices) != 2:
raise ValueError(f"Expected 1 or 2 choices to collapse, got {len(choices)}. ")
for choice in choices:
if choice.message.audio:
if audio_choice is not None:
raise ValueError("Multiple audio choices cannot be set in one completion message")
audio_choice = choice
else:
if content_choice is not None:
raise ValueError("Multiple content choices cannot be set in one completion message")
content_choice = choice
if content_choice is None or audio_choice is None:
raise ValueError(
"Expected one content and one audio choice, but got: "
f"content={content_choice is not None}, audio={audio_choice is not None}"
)
content_choice.message.audio = audio_choice.message.audio
return content_choice
@staticmethod
def _get_subrequest_ids(base_id, request: BatchChatCompletionRequest) -> list[str]:
"""Get the request ID for each entry in the batch request to be resubmitted to chat completions."""
return [f"{base_id}-idx-{idx}" for idx in range(len(request.messages))]
async def create_batch_chat_completion(
self,
request: BatchChatCompletionRequest,
raw_request: Request,
) -> ChatCompletionResponse | ErrorResponse:
"""Given a request, submit each request to chat completions & collect the results."""
model = ""
enabled_streaming = False
chat_requests: list[ChatCompletionRequest] = []
choices: list[ChatCompletionResponseChoice] = []
total_prompt_tokens = 0
total_completion_tokens = 0
# Get the base request from the raw_request before maybe mutating the header
base_id = OmniOpenAIServingChatBatch._base_request_id(
raw_request,
request.request_id,
)
batch_req_id = f"chatcmpl-batch-{base_id}"
sub_req_ids = self._get_subrequest_ids(base_id, request)
# Remove the raw request header if it exists now that we have subreq IDs,
# since this takes priority over the object request_ids
if raw_request.headers.get("X-Request-Id"):
copy_headers = raw_request.headers.mutablecopy()
del copy_headers["X-Request-Id"]
raw_request._headers = copy_headers
for idx, msg in enumerate(request.messages):
# Update the current request ID to avoid subrequest collisions
request.request_id = sub_req_ids[idx]
try:
chat_cmp_request = request.to_chat_completion_request(msg)
except ValidationError as e:
return self._create_error_response(
f"Message {idx} could not be converted to a chat completion request: {e}",
)
# Streaming isn't supported for batch chat completions,
# so always set to False & warn if it was requested.
enabled_streaming |= chat_cmp_request.stream
chat_cmp_request.stream = False
chat_requests.append(chat_cmp_request)
if enabled_streaming:
logger.warning("Streaming is not supported for batched chat completions; ignoring stream=True.")
# Submit each chat completion request as a task, then gather results.
# TODO (Alex): optimize this
tasks = [asyncio.create_task(self.create_chat_completion(c, raw_request)) for c in chat_requests]
try:
results = await asyncio.gather(*tasks)
finally:
# Ensure we cancel remaining tasks if needed, e.g., early exit due to bad behavior
for t in tasks:
if not t.done():
t.cancel()
for i, resp in enumerate(results):
if isinstance(resp, ErrorResponse):
return resp
completion: ChatCompletionResponse = resp
model = completion.model
# FIXME (Alex): We should probably handle this in chat completions,
# not here, but we need to ensure streaming is properly handled
try:
collapsed_choice = self._maybe_collapse_choices(completion.choices)
except ValueError as e:
return self._create_error_response(
f"Failed to collapse choices with error: {e}",
)
collapsed_choice.index = i
choices.append(collapsed_choice)
if completion.usage:
total_prompt_tokens += completion.usage.prompt_tokens
total_completion_tokens += completion.usage.completion_tokens
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
return ChatCompletionResponse(
id=batch_req_id,
created=int(time.time()),
model=model,
choices=choices,
usage=usage,
)