[Frontend] Implement Batched Chat Completions (#5317)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
1 parent
18bf7e0ff9
commit
b15d4e6a64
7 files changed
+531
-39
No files matched your search
@@ -39,6 +39,7 @@ from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, Tool
|
||||
from vllm.entrypoints.openai.api_server import build_app as build_openai_app
|
||||
from vllm.entrypoints.openai.api_server import setup_server as setup_openai_server
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
@@ -95,6 +96,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
from vllm_omni.config.endpoint_policy import shutdown_unsupported_routes
|
||||
from vllm_omni.diffusion.models.interface import ReferenceVideoDecodeSpec
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.openai.batch_serving import OmniOpenAIServingChatBatch
|
||||
from vllm_omni.entrypoints.openai.duplex_capability import should_enable_duplex_endpoint
|
||||
from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError
|
||||
from vllm_omni.entrypoints.openai.image_api_utils import (
|
||||
@@ -513,6 +515,7 @@ async def omni_run_server_worker(listen_address, sock, args, client_config=None,
|
||||
|
||||
# OMNI: Remove upstream routes that we override with omni-specific handlers
|
||||
_remove_route_from_app(app, "/v1/chat/completions", {"POST"})
|
||||
_remove_route_from_app(app, "/v1/chat/completions/batch", {"POST"})
|
||||
_remove_route_from_app(app, "/v1/models", {"GET"}) # Remove upstream /v1/models to use omni's handler
|
||||
app.include_router(router)
|
||||
|
||||
@@ -777,6 +780,10 @@ async def omni_init_app_state(
|
||||
diffusion_engine=engine_client, # type: ignore
|
||||
model_name=model_name,
|
||||
)
|
||||
state.openai_serving_chat_batch = OmniOpenAIServingChatBatch.for_diffusion(
|
||||
diffusion_engine=engine_client, # type: ignore
|
||||
model_name=model_name,
|
||||
)
|
||||
|
||||
# audio related
|
||||
state.openai_serving_speech = None
|
||||
@@ -950,30 +957,33 @@ async def omni_init_app_state(
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.openai_serving_chat = (
|
||||
OmniOpenAIServingChat(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
|
||||
_chat_kwargs = dict(
|
||||
engine_client=engine_client,
|
||||
models=state.openai_serving_models,
|
||||
response_role=args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
)
|
||||
|
||||
state.openai_serving_chat = OmniOpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None
|
||||
state.openai_serving_chat_batch = (
|
||||
OmniOpenAIServingChatBatch(**_chat_kwargs) if "generate" in supported_tasks else None
|
||||
)
|
||||
|
||||
# Warm up chat template processing to avoid first-request latency
|
||||
if state.openai_serving_chat is not None:
|
||||
state.openai_serving_chat.warmup()
|
||||
@@ -1162,6 +1172,10 @@ def Omnichat(request: Request) -> OmniOpenAIServingChat | None:
|
||||
return request.app.state.openai_serving_chat
|
||||
|
||||
|
||||
def OmniBatchChat(request: Request) -> OmniOpenAIServingChatBatch | None:
|
||||
return request.app.state.openai_serving_chat_batch
|
||||
|
||||
|
||||
def Omnispeech(request: Request) -> OmniOpenAIServingSpeech | None:
|
||||
return request.app.state.openai_serving_speech
|
||||
|
||||
@@ -1245,6 +1259,45 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
|
||||
return StreamingResponse(content=generator, media_type="text/event-stream")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/chat/completions/batch",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_batch_chat_completion(request: BatchChatCompletionRequest, raw_request: Request):
|
||||
handler = OmniBatchChat(raw_request)
|
||||
if handler is None:
|
||||
base_server = getattr(raw_request.app.state, "serving_tokenization", None)
|
||||
if base_server is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.NOT_FOUND.value,
|
||||
detail="The model does not support Chat Completions API",
|
||||
)
|
||||
return base_server.create_error_response(message="The model does not support Chat Completions API")
|
||||
try:
|
||||
result = await handler.create_batch_chat_completion(request, raw_request)
|
||||
except (EngineGenerateError, EngineDeadError) as exc:
|
||||
return _create_engine_error_json_response(raw_request, exc)
|
||||
except Exception as e:
|
||||
logger.exception("Batched chat completion failed: %s", e)
|
||||
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(
|
||||
content=result.model_dump(),
|
||||
status_code=result.error.code if result.error else 400,
|
||||
)
|
||||
return JSONResponse(content=result.model_dump(mode="json"))
|
||||
|
||||
|
||||
_remove_route_from_router(router, "/v1/audio/speech", {"POST"})
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ValidationError
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResponseChoice,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
UsageInfo,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OmniOpenAIServingChatBatch(OmniOpenAIServingChat):
|
||||
@staticmethod
|
||||
def _maybe_collapse_choices(choices: list[ChatCompletionResponseChoice]):
|
||||
"""Collapse content + audio choices into one choice.
|
||||
|
||||
NOTE: This is a hack for models that produce separate text and
|
||||
audio choices to ensure we have 1:1 mapping between number of
|
||||
inputs in the batch and responses.
|
||||
"""
|
||||
content_choice, audio_choice = None, None
|
||||
if len(choices) == 1:
|
||||
return choices[0]
|
||||
if len(choices) != 2:
|
||||
raise ValueError(f"Expected 1 or 2 choices to collapse, got {len(choices)}. ")
|
||||
|
||||
for choice in choices:
|
||||
if choice.message.audio:
|
||||
if audio_choice is not None:
|
||||
raise ValueError("Multiple audio choices cannot be set in one completion message")
|
||||
audio_choice = choice
|
||||
else:
|
||||
if content_choice is not None:
|
||||
raise ValueError("Multiple content choices cannot be set in one completion message")
|
||||
content_choice = choice
|
||||
|
||||
if content_choice is None or audio_choice is None:
|
||||
raise ValueError(
|
||||
"Expected one content and one audio choice, but got: "
|
||||
f"content={content_choice is not None}, audio={audio_choice is not None}"
|
||||
)
|
||||
content_choice.message.audio = audio_choice.message.audio
|
||||
return content_choice
|
||||
|
||||
@staticmethod
|
||||
def _get_subrequest_ids(base_id, request: BatchChatCompletionRequest) -> list[str]:
|
||||
"""Get the request ID for each entry in the batch request to be resubmitted to chat completions."""
|
||||
return [f"{base_id}-idx-{idx}" for idx in range(len(request.messages))]
|
||||
|
||||
async def create_batch_chat_completion(
|
||||
self,
|
||||
request: BatchChatCompletionRequest,
|
||||
raw_request: Request,
|
||||
) -> ChatCompletionResponse | ErrorResponse:
|
||||
"""Given a request, submit each request to chat completions & collect the results."""
|
||||
model = ""
|
||||
enabled_streaming = False
|
||||
chat_requests: list[ChatCompletionRequest] = []
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
|
||||
# Get the base request from the raw_request before maybe mutating the header
|
||||
base_id = OmniOpenAIServingChatBatch._base_request_id(
|
||||
raw_request,
|
||||
request.request_id,
|
||||
)
|
||||
batch_req_id = f"chatcmpl-batch-{base_id}"
|
||||
sub_req_ids = self._get_subrequest_ids(base_id, request)
|
||||
|
||||
# Remove the raw request header if it exists now that we have subreq IDs,
|
||||
# since this takes priority over the object request_ids
|
||||
if raw_request.headers.get("X-Request-Id"):
|
||||
copy_headers = raw_request.headers.mutablecopy()
|
||||
del copy_headers["X-Request-Id"]
|
||||
raw_request._headers = copy_headers
|
||||
|
||||
for idx, msg in enumerate(request.messages):
|
||||
# Update the current request ID to avoid subrequest collisions
|
||||
request.request_id = sub_req_ids[idx]
|
||||
try:
|
||||
chat_cmp_request = request.to_chat_completion_request(msg)
|
||||
except ValidationError as e:
|
||||
return self._create_error_response(
|
||||
f"Message {idx} could not be converted to a chat completion request: {e}",
|
||||
)
|
||||
# Streaming isn't supported for batch chat completions,
|
||||
# so always set to False & warn if it was requested.
|
||||
enabled_streaming |= chat_cmp_request.stream
|
||||
chat_cmp_request.stream = False
|
||||
chat_requests.append(chat_cmp_request)
|
||||
|
||||
if enabled_streaming:
|
||||
logger.warning("Streaming is not supported for batched chat completions; ignoring stream=True.")
|
||||
|
||||
# Submit each chat completion request as a task, then gather results.
|
||||
# TODO (Alex): optimize this
|
||||
tasks = [asyncio.create_task(self.create_chat_completion(c, raw_request)) for c in chat_requests]
|
||||
try:
|
||||
results = await asyncio.gather(*tasks)
|
||||
finally:
|
||||
# Ensure we cancel remaining tasks if needed, e.g., early exit due to bad behavior
|
||||
for t in tasks:
|
||||
if not t.done():
|
||||
t.cancel()
|
||||
|
||||
for i, resp in enumerate(results):
|
||||
if isinstance(resp, ErrorResponse):
|
||||
return resp
|
||||
completion: ChatCompletionResponse = resp
|
||||
model = completion.model
|
||||
# FIXME (Alex): We should probably handle this in chat completions,
|
||||
# not here, but we need to ensure streaming is properly handled
|
||||
try:
|
||||
collapsed_choice = self._maybe_collapse_choices(completion.choices)
|
||||
except ValueError as e:
|
||||
return self._create_error_response(
|
||||
f"Failed to collapse choices with error: {e}",
|
||||
)
|
||||
collapsed_choice.index = i
|
||||
choices.append(collapsed_choice)
|
||||
if completion.usage:
|
||||
total_prompt_tokens += completion.usage.prompt_tokens
|
||||
total_completion_tokens += completion.usage.completion_tokens
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=total_prompt_tokens,
|
||||
completion_tokens=total_completion_tokens,
|
||||
total_tokens=total_prompt_tokens + total_completion_tokens,
|
||||
)
|
||||
|
||||
return ChatCompletionResponse(
|
||||
id=batch_req_id,
|
||||
created=int(time.time()),
|
||||
model=model,
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
Reference in new issue
Block a user