272 lines
10 KiB
Python
272 lines
10 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from collections.abc import AsyncGenerator
|
|
from typing import Any
|
|
|
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
|
from vllm.logger import init_logger
|
|
|
|
from vllm_omni.entrypoints.duplex.protocol import DuplexSession
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class ChatFallbackProjectorMixin:
|
|
"""Project generic chat completion output into duplex response events."""
|
|
|
|
async def _run_response(self, session: DuplexSession, send_json) -> None:
|
|
response_id = session.begin_response()
|
|
epoch = session.epoch
|
|
request_id = f"duplex-{session.session_id}-{epoch}-{session.input_commit_seq}"
|
|
session.bind_request(f"chatcmpl-{request_id}")
|
|
await send_json(
|
|
self._response_created_payload(
|
|
session,
|
|
response_id,
|
|
epoch=epoch,
|
|
request_id=session.active_request_id,
|
|
)
|
|
)
|
|
|
|
try:
|
|
request = self._build_chat_request(session, request_id)
|
|
result = await self._chat_service.create_chat_completion(request, raw_request=None)
|
|
if isinstance(result, ErrorResponse):
|
|
error = result.error
|
|
await send_json(
|
|
{
|
|
"type": "error",
|
|
"error": error.message if error else "Chat request failed",
|
|
"code": error.type if error else "chat_error",
|
|
}
|
|
)
|
|
session.end_response(commit_text=False)
|
|
return
|
|
if hasattr(result, "__aiter__"):
|
|
await self._drain_streaming_response(session, result, epoch, response_id, send_json)
|
|
else:
|
|
await self._emit_full_response(session, result, epoch, response_id, send_json)
|
|
if session.epoch == epoch:
|
|
final_stage_metrics = session.accumulate_response_stage_metrics(None)
|
|
should_commit = self._should_commit_response_to_history(session, response_id)
|
|
committed_message = session.end_response(commit_text=should_commit)
|
|
if should_commit:
|
|
session.register_history_item(f"item_{response_id}", committed_message)
|
|
done_payload: dict[str, object] = {
|
|
"type": "response.done",
|
|
"session_id": session.session_id,
|
|
"response_id": response_id,
|
|
"epoch": epoch,
|
|
"committed": committed_message is not None,
|
|
"playback": session.playback.as_dict(),
|
|
}
|
|
if final_stage_metrics:
|
|
done_payload["vllm_omni"] = {"stage_metrics": final_stage_metrics}
|
|
await send_json(done_payload)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.exception("Duplex response failed: %s", exc)
|
|
session.end_response(commit_text=False)
|
|
await send_json(
|
|
{
|
|
"type": "error",
|
|
"session_id": session.session_id,
|
|
"response_id": response_id,
|
|
"error": str(exc),
|
|
"code": "response_error",
|
|
}
|
|
)
|
|
|
|
def _build_chat_request(self, session: DuplexSession, request_id: str) -> ChatCompletionRequest:
|
|
response_config = session.response_config
|
|
messages: list[dict[str, object]] = []
|
|
if response_config.instructions:
|
|
messages.append({"role": "system", "content": response_config.instructions})
|
|
messages.extend(session.history)
|
|
|
|
kwargs: dict[str, Any] = {
|
|
"model": response_config.model or self._chat_service.model_config.model,
|
|
"messages": messages,
|
|
"stream": True,
|
|
}
|
|
if response_config.temperature is not None:
|
|
kwargs["temperature"] = response_config.temperature
|
|
if response_config.max_tokens is not None:
|
|
kwargs["max_tokens"] = response_config.max_tokens
|
|
model_extra = dict(response_config.extra_body)
|
|
model_extra.pop("native_duplex", None)
|
|
model_extra.pop("minicpmo45_native_duplex", None)
|
|
tools = model_extra.pop("realtime_response_tools", model_extra.pop("realtime_tools", None))
|
|
tool_choice = model_extra.pop(
|
|
"realtime_response_tool_choice",
|
|
model_extra.pop("realtime_tool_choice", None),
|
|
)
|
|
for protocol_key in (
|
|
"realtime_response_conversation",
|
|
"realtime_response_metadata",
|
|
"realtime_response_prompt",
|
|
):
|
|
model_extra.pop(protocol_key, None)
|
|
kwargs.update(model_extra)
|
|
if isinstance(tools, list):
|
|
kwargs["tools"] = tools
|
|
if isinstance(tool_choice, str | dict):
|
|
kwargs["tool_choice"] = tool_choice
|
|
|
|
request = ChatCompletionRequest(**kwargs)
|
|
object.__setattr__(request, "modalities", response_config.modalities)
|
|
object.__setattr__(request, "request_id", request_id)
|
|
object.__setattr__(
|
|
request,
|
|
"chat_template_kwargs",
|
|
{"use_tts_template": response_config.use_tts_template},
|
|
)
|
|
return request
|
|
|
|
async def _drain_streaming_response(
|
|
self,
|
|
session: DuplexSession,
|
|
result: AsyncGenerator[str, None],
|
|
epoch: int,
|
|
response_id: str,
|
|
send_json,
|
|
) -> None:
|
|
async for raw_chunk in result:
|
|
if session.epoch != epoch:
|
|
return
|
|
for payload in self._parse_sse_payloads(raw_chunk):
|
|
if payload == "[DONE]":
|
|
continue
|
|
if isinstance(payload, dict):
|
|
await self._emit_chat_payload(session, payload, epoch, response_id, send_json)
|
|
|
|
async def _emit_full_response(
|
|
self,
|
|
session: DuplexSession,
|
|
result: Any,
|
|
epoch: int,
|
|
response_id: str,
|
|
send_json,
|
|
) -> None:
|
|
if hasattr(result, "model_dump"):
|
|
payload = result.model_dump(mode="json", exclude_unset=True)
|
|
else:
|
|
payload = {"response": str(result)}
|
|
await self._emit_chat_payload(session, payload, epoch, response_id, send_json)
|
|
|
|
def _parse_sse_payloads(self, raw_chunk: str) -> list[dict[str, object] | str]:
|
|
payloads: list[dict[str, object] | str] = []
|
|
for line in raw_chunk.splitlines():
|
|
line = line.strip()
|
|
if not line.startswith("data:"):
|
|
continue
|
|
data = line[5:].strip()
|
|
if not data:
|
|
continue
|
|
if data == "[DONE]":
|
|
payloads.append(data)
|
|
continue
|
|
try:
|
|
parsed = json.loads(data)
|
|
except json.JSONDecodeError:
|
|
logger.debug("Skipping non-JSON duplex stream payload: %s", data)
|
|
continue
|
|
if isinstance(parsed, dict):
|
|
payloads.append(parsed)
|
|
return payloads
|
|
|
|
async def _emit_chat_payload(
|
|
self,
|
|
session: DuplexSession,
|
|
payload: dict[str, object],
|
|
epoch: int,
|
|
response_id: str,
|
|
send_json,
|
|
) -> None:
|
|
metrics = payload.get("metrics")
|
|
stage_metrics = metrics.get("stage_metrics") if isinstance(metrics, dict) else None
|
|
if isinstance(stage_metrics, dict):
|
|
session.replace_response_stage_metric_snapshots(stage_metrics)
|
|
|
|
modality = payload.get("modality")
|
|
if modality not in {None, "text", "audio"}:
|
|
await send_json(
|
|
{
|
|
"type": "error",
|
|
"session_id": session.session_id,
|
|
"response_id": response_id,
|
|
"epoch": epoch,
|
|
"code": "unsupported_response_modality",
|
|
"error": f"Unsupported chat response modality: {modality}",
|
|
}
|
|
)
|
|
return
|
|
choices = payload.get("choices")
|
|
if not isinstance(choices, list):
|
|
await send_json(
|
|
{
|
|
"type": "response.message",
|
|
"session_id": session.session_id,
|
|
"response_id": response_id,
|
|
"epoch": epoch,
|
|
"payload": payload,
|
|
}
|
|
)
|
|
return
|
|
|
|
for choice in choices:
|
|
if not isinstance(choice, dict):
|
|
continue
|
|
delta = choice.get("delta")
|
|
message = choice.get("message")
|
|
content = None
|
|
if isinstance(delta, dict):
|
|
content = delta.get("content")
|
|
elif isinstance(message, dict):
|
|
content = message.get("content")
|
|
|
|
if isinstance(content, str) and content:
|
|
if modality == "audio":
|
|
session.mark_audio_sent()
|
|
await send_json(
|
|
{
|
|
"type": "response.output_audio.delta",
|
|
"session_id": session.session_id,
|
|
"response_id": response_id,
|
|
"epoch": epoch,
|
|
"audio": content,
|
|
"format": session.response_config.response_format,
|
|
}
|
|
)
|
|
else:
|
|
session.append_assistant_text(content)
|
|
await send_json(
|
|
{
|
|
"type": "response.text.delta",
|
|
"session_id": session.session_id,
|
|
"response_id": response_id,
|
|
"epoch": epoch,
|
|
"delta": content,
|
|
}
|
|
)
|
|
|
|
finish_reason = choice.get("finish_reason")
|
|
if finish_reason is not None and modality != "audio":
|
|
await send_json(
|
|
{
|
|
"type": "response.output_item.done",
|
|
"session_id": session.session_id,
|
|
"response_id": response_id,
|
|
"epoch": epoch,
|
|
"finish_reason": finish_reason,
|
|
"modality": modality,
|
|
}
|
|
)
|