1899 lines
79 KiB
Python
1899 lines
79 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import binascii
|
|
import json
|
|
import time
|
|
import warnings
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import numpy as np
|
|
import pybase64 as base64
|
|
from fastapi import WebSocket, WebSocketDisconnect
|
|
from openai.types import realtime as types
|
|
from pydantic import TypeAdapter
|
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
|
ChatCompletionNamedToolChoiceParam,
|
|
ChatCompletionRequest,
|
|
ChatCompletionToolsParam,
|
|
)
|
|
from vllm.logger import init_logger
|
|
from vllm.sampling_params import RequestOutputKind, SamplingParams, StructuredOutputsParams
|
|
from vllm.tool_parsers import ToolParserManager
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.inputs import TokensPrompt
|
|
|
|
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
|
from vllm_omni.entrypoints.openai.realtime.session import (
|
|
ActiveResponse,
|
|
AudioFullDuplexSessionState,
|
|
HistoryLimitError,
|
|
ResponseUsage,
|
|
_gen_id,
|
|
merge_session_config,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
_CLIENT_EVENT_ADAPTER = TypeAdapter(types.RealtimeClientEvent)
|
|
|
|
# Qwen3-Omni's Realtime contract is mono signed PCM16 at 24 kHz. PCM16 is the
|
|
# sample width; 24 kHz is the sample rate.
|
|
SAMPLE_RATE_HZ = 24000
|
|
BYTES_PER_SAMPLE_PCM16 = 2
|
|
# Application safety caps for one append and the complete pending input turn.
|
|
MAX_AUDIO_APPEND_BYTES = 15 * 1024 * 1024
|
|
MAX_INPUT_AUDIO_BUFFER_BYTES = 64 * 1024 * 1024
|
|
|
|
# The Qwen3-Omni chat template uses these literal special tokens for audio.
|
|
# The generic chat-template interface does not expose that placeholder as
|
|
# metadata, so it cannot be inferred reliably from the Realtime event schema.
|
|
AUDIO_PLACEHOLDER = "<|audio_start|><|audio_pad|><|audio_end|>"
|
|
|
|
# Empirically calibrated for Qwen3-Omni from 8,808 ms / 23 thinker tokens.
|
|
QWEN3_OMNI_MS_PER_TOKEN = 383.0
|
|
|
|
AUTO_TRUNCATION_TRIGGER_RATIO = 0.8
|
|
AUTO_TRUNCATION_TARGET_RATIO = 0.5
|
|
|
|
|
|
class _UnsupportedAudioFormatError(ValueError):
|
|
"""The Realtime endpoint only accepts mono PCM16 audio at 24 kHz."""
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _ResolvedResponse:
|
|
input: list[Any] | None
|
|
instructions: str | None
|
|
modalities: list[str]
|
|
max_output_tokens: int | str
|
|
tools: list[Any] | None
|
|
tool_choice: Any
|
|
metadata: Any
|
|
|
|
|
|
class OpenAIFullDuplexConnection:
|
|
"""Handle one OpenAI Realtime WebSocket session."""
|
|
|
|
def __init__(
|
|
self,
|
|
websocket: WebSocket,
|
|
engine: AsyncOmni,
|
|
model_name: str,
|
|
tokenizer: Any,
|
|
tool_call_parser: str | None = None,
|
|
enable_auto_tool_choice: bool = False,
|
|
):
|
|
self.ws = websocket
|
|
self.engine = engine
|
|
self.model_name = model_name
|
|
self._tool_call_parser_name = tool_call_parser if enable_auto_tool_choice else None
|
|
|
|
self.session = AudioFullDuplexSessionState()
|
|
self.session.config.model = model_name
|
|
|
|
self._connected = True
|
|
self._response_task: asyncio.Task | None = None
|
|
self._response_cancel_event = asyncio.Event()
|
|
self._send_lock = asyncio.Lock()
|
|
|
|
self._tokenizer = self._resolve_tokenizer(tokenizer)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Lifecycle #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def handle_connection(self):
|
|
try:
|
|
await self.ws.accept()
|
|
logger.info("[realtime] connection opened, session_id=%s", self.session.session_id)
|
|
await self._send_session_created()
|
|
await self._send_conversation_created()
|
|
|
|
while self._connected:
|
|
try:
|
|
text = await self.ws.receive_text()
|
|
except WebSocketDisconnect:
|
|
break
|
|
try:
|
|
event = _CLIENT_EVENT_ADAPTER.validate_json(text)
|
|
except Exception:
|
|
await self._send_error(
|
|
"Invalid or unrecognized client event",
|
|
"invalid_event",
|
|
)
|
|
continue
|
|
await self._dispatch_event(event)
|
|
except Exception:
|
|
logger.exception("Unhandled error in realtime connection")
|
|
await self._send_error("Internal server error", "server_error", error_type="server_error")
|
|
finally:
|
|
await self._cleanup()
|
|
|
|
async def _cleanup(self):
|
|
self._connected = False
|
|
await self._cancel_active_response()
|
|
logger.info("[realtime] connection closed, session_id=%s", self.session.session_id)
|
|
|
|
def _resolve_tokenizer(self, tokenizer: Any) -> Any:
|
|
if getattr(tokenizer, "chat_template", None):
|
|
return tokenizer
|
|
|
|
input_processor = getattr(self.engine, "input_processor", None)
|
|
if input_processor is None:
|
|
return tokenizer
|
|
try:
|
|
from vllm.transformers_utils.processor import cached_processor_from_config
|
|
|
|
processor = cached_processor_from_config(input_processor.model_config)
|
|
if getattr(processor, "apply_chat_template", None):
|
|
return processor
|
|
except Exception:
|
|
logger.warning("Could not load processor for chat templating")
|
|
return tokenizer
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Event dispatch #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _dispatch_event(self, event: types.RealtimeClientEvent):
|
|
match event:
|
|
case types.SessionUpdateEvent():
|
|
handler = self._handle_session_update
|
|
case types.InputAudioBufferAppendEvent():
|
|
handler = self._handle_audio_append
|
|
case types.InputAudioBufferCommitEvent():
|
|
handler = self._handle_audio_commit
|
|
case types.InputAudioBufferClearEvent():
|
|
handler = self._handle_audio_clear
|
|
case types.ResponseCreateEvent():
|
|
handler = self._handle_response_create
|
|
case types.ResponseCancelEvent():
|
|
handler = self._handle_response_cancel
|
|
case types.ConversationItemCreateEvent():
|
|
handler = self._handle_item_create
|
|
case types.ConversationItemDeleteEvent():
|
|
handler = self._handle_item_delete
|
|
case types.ConversationItemRetrieveEvent():
|
|
handler = self._handle_item_retrieve
|
|
case types.ConversationItemTruncateEvent():
|
|
handler = self._handle_item_truncate
|
|
case _:
|
|
await self._send_error(
|
|
f"Unknown event type: {event.type}",
|
|
"invalid_event",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
try:
|
|
await handler(event)
|
|
except Exception:
|
|
logger.exception("Error handling event %s", event.type)
|
|
await self._send_error(
|
|
"Internal server error",
|
|
"server_error",
|
|
event_id=event.event_id,
|
|
error_type="server_error",
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# session.update #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _handle_session_update(self, event: types.SessionUpdateEvent):
|
|
s = self.session
|
|
try:
|
|
cfg = self._sanitize_session_config(event.session)
|
|
except _UnsupportedAudioFormatError as exc:
|
|
await self._send_error(str(exc), "unsupported_audio_format", event_id=event.event_id)
|
|
return
|
|
s.config = merge_session_config(s.config, cfg)
|
|
await self._send_session_updated()
|
|
|
|
def _sanitize_session_config(self, cfg: Any) -> types.RealtimeSessionCreateRequest:
|
|
"""Drop any session.update fields this server doesn't support instead
|
|
of rejecting the whole update, so clients that always send their full
|
|
default config (transcription, noise reduction, server-side VAD, MCP
|
|
tools, ...) still get a working session rather than a hard error.
|
|
"""
|
|
data = cfg.model_dump(exclude_unset=True)
|
|
|
|
if "model" in data and data["model"] != self.model_name:
|
|
data.pop("model")
|
|
|
|
if "tools" in data:
|
|
data["tools"] = [t for t in data["tools"] if (t or {}).get("type") != "mcp"]
|
|
if (data.get("tool_choice") or {}).get("type") == "mcp" if isinstance(data.get("tool_choice"), dict) else False:
|
|
data.pop("tool_choice")
|
|
|
|
audio = data.get("audio")
|
|
if audio is not None:
|
|
inp = audio.get("input")
|
|
if inp is not None:
|
|
inp.pop("transcription", None)
|
|
inp.pop("noise_reduction", None)
|
|
if not self._is_pcm16_24khz_format(inp.get("format")):
|
|
raise _UnsupportedAudioFormatError("Only 24 kHz PCM16 input audio is supported")
|
|
td = inp.get("turn_detection")
|
|
if isinstance(td, dict) and td.get("type") in ("server_vad", "semantic_vad"):
|
|
inp["turn_detection"] = None
|
|
|
|
out = audio.get("output")
|
|
if out is not None:
|
|
if out.get("speed") not in (None, 1):
|
|
out.pop("speed", None)
|
|
if not self._is_pcm16_24khz_format(out.get("format")):
|
|
raise _UnsupportedAudioFormatError("Only 24 kHz PCM16 output audio is supported")
|
|
|
|
return types.RealtimeSessionCreateRequest.model_validate(data)
|
|
|
|
@staticmethod
|
|
def _unsupported_audio_option(audio: Any) -> str | None:
|
|
if audio is None:
|
|
return None
|
|
audio_input = getattr(audio, "input", None)
|
|
if audio_input is not None:
|
|
fields: set[str] = getattr(audio_input, "model_fields_set", set())
|
|
if "transcription" in fields and audio_input.transcription is not None:
|
|
return "Input audio transcription"
|
|
if "noise_reduction" in fields and audio_input.noise_reduction is not None:
|
|
return "Input audio noise reduction"
|
|
output = getattr(audio, "output", None)
|
|
if output is not None:
|
|
fields = getattr(output, "model_fields_set", set())
|
|
if "speed" in fields and output.speed is not None and output.speed != 1:
|
|
return "Output audio speed"
|
|
return None
|
|
|
|
@staticmethod
|
|
def _is_pcm16_24khz_format(audio_format: Any) -> bool:
|
|
if audio_format is None:
|
|
return True
|
|
if isinstance(audio_format, str):
|
|
return audio_format in ("audio/pcm", "pcm16")
|
|
if isinstance(audio_format, dict):
|
|
format_type = audio_format.get("type")
|
|
rate = audio_format.get("rate", SAMPLE_RATE_HZ)
|
|
else:
|
|
format_type = getattr(audio_format, "type", None)
|
|
rate = getattr(audio_format, "rate", SAMPLE_RATE_HZ)
|
|
return format_type in ("audio/pcm", "pcm16") and rate == SAMPLE_RATE_HZ
|
|
|
|
@staticmethod
|
|
def _uses_mcp(config: Any) -> bool:
|
|
def object_type(value: Any) -> Any:
|
|
return value.get("type") if isinstance(value, dict) else getattr(value, "type", None)
|
|
|
|
return (
|
|
any(object_type(tool) == "mcp" for tool in (getattr(config, "tools", None) or []))
|
|
or object_type(getattr(config, "tool_choice", None)) == "mcp"
|
|
)
|
|
|
|
async def _send_unsupported_mcp(self, event_id: str | None) -> None:
|
|
await self._send_error(
|
|
"Remote MCP tools are not supported",
|
|
"unsupported_feature",
|
|
event_id=event_id,
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# input_audio_buffer.append / .commit / .clear #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _handle_audio_append(self, event: types.InputAudioBufferAppendEvent):
|
|
if not event.audio:
|
|
return
|
|
try:
|
|
audio_bytes = self._decode_pcm16(event.audio)
|
|
if len(self.session.input_audio_buffer) + len(audio_bytes) > MAX_INPUT_AUDIO_BUFFER_BYTES:
|
|
limit_mib = MAX_INPUT_AUDIO_BUFFER_BYTES // (1024 * 1024)
|
|
raise ValueError(f"Input audio buffer exceeds the {limit_mib} MiB limit")
|
|
except ValueError as exc:
|
|
await self._send_error(str(exc), "invalid_request_error", event_id=event.event_id)
|
|
return
|
|
self.session.input_audio_buffer.extend(audio_bytes)
|
|
|
|
@staticmethod
|
|
def _decode_pcm16(audio: str, max_bytes: int = MAX_AUDIO_APPEND_BYTES) -> bytes:
|
|
if len(audio) > 4 * ((max_bytes + 2) // 3):
|
|
raise ValueError("Audio payload is too large")
|
|
try:
|
|
decoded = base64.b64decode(audio, validate=True)
|
|
except (ValueError, binascii.Error) as exc:
|
|
raise ValueError("Invalid base64 audio data") from exc
|
|
if len(decoded) > max_bytes:
|
|
raise ValueError("Audio payload is too large")
|
|
if len(decoded) % BYTES_PER_SAMPLE_PCM16:
|
|
raise ValueError("PCM audio data must contain complete 16-bit samples")
|
|
return decoded
|
|
|
|
async def _commit_audio_buffer(
|
|
self,
|
|
event_id: str | None = None,
|
|
) -> types.RealtimeConversationItemUserMessage | None:
|
|
"""Commit buffered audio and announce the conversation item."""
|
|
s = self.session
|
|
if len(s.input_audio_buffer) == 0:
|
|
return None
|
|
pcm16 = np.frombuffer(bytes(s.input_audio_buffer), dtype=np.int16)
|
|
audio_f32 = pcm16.astype(np.float32) / 32768.0
|
|
item = types.RealtimeConversationItemUserMessage(
|
|
type="message",
|
|
role="user",
|
|
status="completed",
|
|
content=[{"type": "input_audio", "audio": self._pcm16_b64(audio_f32)}],
|
|
)
|
|
try:
|
|
s.insert_item(item)
|
|
except HistoryLimitError as exc:
|
|
s.input_audio_buffer.clear()
|
|
await self._send_error(str(exc), "invalid_request_error", event_id=event_id)
|
|
return None
|
|
s.input_audio_buffer.clear()
|
|
idx = self.session.find_item_index(item.id)
|
|
previous_item_id = self.session.items[idx - 1].id if idx else None
|
|
await self._send_event(
|
|
types.InputAudioBufferCommittedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="input_audio_buffer.committed",
|
|
item_id=item.id,
|
|
previous_item_id=previous_item_id,
|
|
)
|
|
)
|
|
# Do not echo the client's potentially large audio payload.
|
|
wire_item = item.model_copy(update={"content": []})
|
|
await self._send_conversation_item_added_and_done(wire_item, previous_item_id)
|
|
return item
|
|
|
|
async def _handle_audio_commit(self, event: types.InputAudioBufferCommitEvent):
|
|
s = self.session
|
|
if len(s.input_audio_buffer) == 0:
|
|
await self._send_error(
|
|
"Input audio buffer is empty",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
|
|
await self._commit_audio_buffer(event.event_id)
|
|
|
|
async def _handle_audio_clear(self, event: types.InputAudioBufferClearEvent):
|
|
self.session.input_audio_buffer.clear()
|
|
await self._send_event(
|
|
types.InputAudioBufferClearedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="input_audio_buffer.cleared",
|
|
)
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# response.create #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
@staticmethod
|
|
def _resolve_tools_and_choice(s: AudioFullDuplexSessionState, response_cfg: Any):
|
|
"""Per-response overrides win over session.config, same pattern as
|
|
output_modalities/max_output_tokens below."""
|
|
tools = s.config.tools
|
|
if response_cfg is not None and getattr(response_cfg, "tools", None) is not None:
|
|
tools = response_cfg.tools
|
|
tool_choice = s.config.tool_choice
|
|
if response_cfg is not None and getattr(response_cfg, "tool_choice", None) is not None:
|
|
tool_choice = response_cfg.tool_choice
|
|
return tools, tool_choice
|
|
|
|
def _resolve_response(self, response_cfg: Any) -> _ResolvedResponse:
|
|
s = self.session
|
|
tools, tool_choice = self._resolve_tools_and_choice(s, response_cfg)
|
|
modalities = s.config.output_modalities
|
|
max_output_tokens = s.config.max_output_tokens
|
|
response_input = None
|
|
instructions = None
|
|
metadata = None
|
|
if response_cfg is not None:
|
|
if response_cfg.output_modalities is not None:
|
|
modalities = response_cfg.output_modalities
|
|
if response_cfg.max_output_tokens is not None:
|
|
max_output_tokens = response_cfg.max_output_tokens
|
|
response_input = response_cfg.input
|
|
instructions = response_cfg.instructions
|
|
if response_cfg.conversation == "none":
|
|
raise ValueError("Out-of-band responses are not supported")
|
|
metadata = response_cfg.metadata
|
|
audio = getattr(response_cfg, "audio", None)
|
|
unsupported = self._unsupported_audio_option(audio)
|
|
if unsupported is not None:
|
|
raise ValueError(f"{unsupported} is not supported")
|
|
output = getattr(audio, "output", None) if audio is not None else None
|
|
if output is not None and not self._is_pcm16_24khz_format(getattr(output, "format", None)):
|
|
raise _UnsupportedAudioFormatError("Only 24 kHz PCM16 output audio is supported")
|
|
|
|
if tools and tool_choice != "none" and self._tool_call_parser_name is None:
|
|
raise ValueError("Function tools require --enable-auto-tool-choice and --tool-call-parser")
|
|
|
|
resolved_input = self._resolve_response_input(response_input) if response_input is not None else None
|
|
return _ResolvedResponse(
|
|
input=resolved_input,
|
|
instructions=instructions,
|
|
modalities=list(modalities),
|
|
max_output_tokens=max_output_tokens,
|
|
tools=tools,
|
|
tool_choice=tool_choice,
|
|
metadata=metadata,
|
|
)
|
|
|
|
async def _estimate_total_tokens(
|
|
self,
|
|
tools: list | None,
|
|
*,
|
|
instructions: str | None = None,
|
|
items: list[Any] | None = None,
|
|
) -> int:
|
|
t0 = time.monotonic()
|
|
prompt = await self._build_full_prompt(tools=tools, instructions=instructions, items=items)
|
|
total = len(prompt["prompt_token_ids"])
|
|
audio_arrays = prompt.get("multi_modal_data", {}).get("audio", [])
|
|
if audio_arrays:
|
|
raw_tok = getattr(self._tokenizer, "tokenizer", self._tokenizer)
|
|
placeholder_len = len(raw_tok.encode(AUDIO_PLACEHOLDER, add_special_tokens=False))
|
|
for arr, sr in audio_arrays:
|
|
total += self._qwen3_omni_audio_token_count(arr.shape[0], sr) - placeholder_len
|
|
elapsed = time.monotonic() - t0
|
|
if elapsed > 0.05:
|
|
logger.warning(
|
|
"[realtime] _estimate_total_tokens took %.3fs (history=%d items, %d audio segments)",
|
|
elapsed,
|
|
len(items if items is not None else self.session.items),
|
|
len(audio_arrays),
|
|
)
|
|
return total
|
|
|
|
async def _maybe_truncate_history(self, response: _ResolvedResponse) -> bool:
|
|
s = self.session
|
|
max_model_len = getattr(self.engine.model_config, "max_model_len", None)
|
|
if not max_model_len:
|
|
return True
|
|
|
|
truncation = s.config.truncation or "auto"
|
|
ratio = 1.0
|
|
custom_limit = None
|
|
if isinstance(truncation, str):
|
|
mode = "disabled" if truncation == "disabled" else "auto"
|
|
else:
|
|
mode = "retention_ratio"
|
|
ratio = truncation.retention_ratio
|
|
if truncation.token_limits is not None:
|
|
custom_limit = truncation.token_limits.post_instructions
|
|
|
|
reserved_output = response.max_output_tokens if isinstance(response.max_output_tokens, int) else 0
|
|
limit = custom_limit if custom_limit is not None else max(0, max_model_len - reserved_output)
|
|
|
|
if mode == "auto":
|
|
trigger = int(limit * AUTO_TRUNCATION_TRIGGER_RATIO)
|
|
target = int(limit * AUTO_TRUNCATION_TARGET_RATIO)
|
|
elif mode == "retention_ratio":
|
|
trigger = limit
|
|
target = int(limit * ratio)
|
|
else: # disabled
|
|
trigger = limit
|
|
target = limit
|
|
|
|
persistent = response.input is None
|
|
items: list[Any] = s.items if response.input is None else response.input
|
|
total = await self._estimate_total_tokens(
|
|
response.tools,
|
|
instructions=response.instructions,
|
|
items=items,
|
|
)
|
|
if total <= trigger:
|
|
return True
|
|
if mode == "disabled":
|
|
logger.warning(
|
|
"[realtime] token budget exceeded (%d/%d) and truncation is disabled -- rejecting response.create",
|
|
total,
|
|
limit,
|
|
)
|
|
return False
|
|
|
|
idx = 0
|
|
while total > target and idx < len(items):
|
|
item = items[idx]
|
|
if getattr(item, "role", None) == "system":
|
|
idx += 1
|
|
continue
|
|
if persistent and item.id is not None and s.item_in_progress.get(item.id, False):
|
|
idx += 1
|
|
continue
|
|
|
|
remove_indexes = [idx]
|
|
if item.type in ("function_call", "function_call_output"):
|
|
call_id = getattr(item, "call_id", None)
|
|
pair_idx = next(
|
|
(
|
|
other_idx
|
|
for other_idx, other in enumerate(items)
|
|
if other_idx != idx and getattr(other, "call_id", None) == call_id
|
|
),
|
|
None,
|
|
)
|
|
if pair_idx is not None:
|
|
remove_indexes.append(pair_idx)
|
|
|
|
for remove_idx in sorted(remove_indexes, reverse=True):
|
|
removed = items[remove_idx]
|
|
if persistent:
|
|
s.remove_item(removed.id)
|
|
await self._send_event(
|
|
types.ConversationItemDeletedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="conversation.item.deleted",
|
|
item_id=removed.id,
|
|
)
|
|
)
|
|
else:
|
|
del items[remove_idx]
|
|
total = await self._estimate_total_tokens(
|
|
response.tools,
|
|
instructions=response.instructions,
|
|
items=items,
|
|
)
|
|
|
|
return total <= limit
|
|
|
|
async def _handle_response_create(self, event: types.ResponseCreateEvent):
|
|
s = self.session
|
|
response_cfg = event.response
|
|
|
|
if response_cfg is not None and self._uses_mcp(response_cfg):
|
|
await self._send_unsupported_mcp(event.event_id)
|
|
return
|
|
|
|
try:
|
|
response = self._resolve_response(response_cfg)
|
|
except _UnsupportedAudioFormatError as exc:
|
|
await self._send_error(str(exc), "unsupported_audio_format", event_id=event.event_id)
|
|
return
|
|
except ValueError as exc:
|
|
await self._send_error(str(exc), "invalid_request_error", event_id=event.event_id)
|
|
return
|
|
|
|
if not await self._maybe_truncate_history(response):
|
|
await self._send_error(
|
|
"The response input exceeds the model's input token limit",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
|
|
response_id = _gen_id("resp")
|
|
await self._send_event(
|
|
types.ResponseCreatedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.created",
|
|
response=self._response_object(response_id, response, "in_progress"),
|
|
)
|
|
)
|
|
|
|
if s.active_response is not None:
|
|
await self._cancel_active_response()
|
|
|
|
s.active_response = ActiveResponse(response_id=response_id, request_id=f"rt-{response_id}")
|
|
|
|
self._response_cancel_event.clear()
|
|
self._response_task = asyncio.create_task(self._run_response(response_id, response))
|
|
|
|
def _response_object(
|
|
self,
|
|
response_id: str,
|
|
response: _ResolvedResponse,
|
|
status: str,
|
|
*,
|
|
output: list[Any] | None = None,
|
|
status_details: Any = None,
|
|
usage: Any = None,
|
|
) -> types.RealtimeResponse:
|
|
return types.RealtimeResponse(
|
|
id=response_id,
|
|
object="realtime.response",
|
|
status=status,
|
|
status_details=status_details,
|
|
output=output or [],
|
|
conversation_id=self.session.conversation_id,
|
|
output_modalities=response.modalities,
|
|
max_output_tokens=response.max_output_tokens,
|
|
metadata=response.metadata,
|
|
usage=usage,
|
|
)
|
|
|
|
async def _run_response(self, response_id: str, response: _ResolvedResponse):
|
|
s = self.session
|
|
active = s.active_response
|
|
if active is None:
|
|
return
|
|
|
|
completed = False
|
|
try:
|
|
await self._run_response_inner(response_id, response, s, active)
|
|
completed = True
|
|
except asyncio.CancelledError:
|
|
if self._response_cancel_event.is_set():
|
|
await self._finish_cancelled_response(s, response_id, response, active)
|
|
completed = True
|
|
else:
|
|
raise
|
|
except Exception:
|
|
logger.exception("_run_response failed for %s", response_id)
|
|
finally:
|
|
if not completed:
|
|
await self._fail_response(s, response_id, response, active.item_id)
|
|
s.active_response = None
|
|
|
|
def _mark_terminal_response(self, response_id: str) -> bool:
|
|
active = self.session.active_response
|
|
if active is None or active.response_id != response_id or active.terminal_event_sent:
|
|
return False
|
|
active.terminal_event_sent = True
|
|
return True
|
|
|
|
async def _finish_cancelled_response(
|
|
self,
|
|
s: AudioFullDuplexSessionState,
|
|
response_id: str,
|
|
response: _ResolvedResponse,
|
|
active: ActiveResponse,
|
|
) -> None:
|
|
if active.item_id is not None and s.item_in_progress.pop(active.item_id, False):
|
|
s.pending_truncations_ms.pop(active.item_id, None)
|
|
s.remove_item(active.item_id)
|
|
if not self._mark_terminal_response(response_id):
|
|
return
|
|
await self._send_event(
|
|
types.ResponseDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.done",
|
|
response=self._response_object(
|
|
response_id,
|
|
response,
|
|
"cancelled",
|
|
status_details={"type": "cancelled", "reason": "client_cancelled"},
|
|
),
|
|
)
|
|
)
|
|
|
|
async def _fail_response(
|
|
self,
|
|
s: AudioFullDuplexSessionState,
|
|
response_id: str,
|
|
response: _ResolvedResponse,
|
|
item_id: str | None,
|
|
) -> None:
|
|
if item_id is not None and s.item_in_progress.pop(item_id, False):
|
|
s.pending_truncations_ms.pop(item_id, None)
|
|
s.remove_item(item_id)
|
|
if not self._mark_terminal_response(response_id):
|
|
return
|
|
try:
|
|
await self._send_event(
|
|
types.ResponseDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.done",
|
|
response=self._response_object(
|
|
response_id,
|
|
response,
|
|
"failed",
|
|
status_details={
|
|
"type": "failed",
|
|
"error": {"type": "server_error", "code": None},
|
|
},
|
|
),
|
|
)
|
|
)
|
|
except Exception:
|
|
logger.debug("Failed to send failure response.done for %s", response_id, exc_info=True)
|
|
|
|
async def _run_response_inner(self, response_id, response, s, active):
|
|
previous_item_id = s.items[-1].id if s.items else None
|
|
modalities = response.modalities
|
|
is_audio = "audio" in modalities
|
|
tools, tool_choice = response.tools, response.tool_choice
|
|
prompt = await self._build_full_prompt(
|
|
tools=tools,
|
|
instructions=response.instructions,
|
|
items=response.input,
|
|
)
|
|
|
|
item_id = _gen_id("item")
|
|
active.item_id = item_id
|
|
output_index = 0
|
|
|
|
converted_tools = self._convert_tools(tools) if tools else []
|
|
tool_parser = None
|
|
structural_tag_json = None
|
|
if converted_tools and tool_choice != "none" and self._tool_call_parser_name:
|
|
tool_parser_cls = ToolParserManager.get_tool_parser(self._tool_call_parser_name)
|
|
strict_tools = [ChatCompletionToolsParam(**t) for t in self._convert_tools(tools, strict=True)]
|
|
tool_parser = tool_parser_cls(self._tokenizer, tools=strict_tools)
|
|
parser_request = ChatCompletionRequest(
|
|
messages=[],
|
|
tools=strict_tools,
|
|
tool_choice=self._convert_tool_choice(tool_choice),
|
|
include_reasoning=False,
|
|
skip_special_tokens=True,
|
|
)
|
|
structure_tag = tool_parser.get_structural_tag(
|
|
parser_request,
|
|
reasoning=False,
|
|
)
|
|
if structure_tag is not None:
|
|
structural_tag_json = json.dumps(structure_tag.model_dump())
|
|
|
|
item_obj = types.RealtimeConversationItemAssistantMessage(
|
|
type="message",
|
|
role="assistant",
|
|
id=item_id,
|
|
status="in_progress",
|
|
content=[],
|
|
)
|
|
|
|
try:
|
|
s.insert_item(item_obj, previous_item_id=previous_item_id or "root")
|
|
except ValueError:
|
|
logger.warning(
|
|
"[realtime] previous item '%s' vanished while starting response %s; appending instead",
|
|
previous_item_id,
|
|
response_id,
|
|
)
|
|
s.insert_item(item_obj)
|
|
s.item_in_progress[item_id] = True
|
|
|
|
await self._send_event(
|
|
types.ResponseOutputItemAddedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_item.added",
|
|
response_id=response_id,
|
|
output_index=output_index,
|
|
item=item_obj, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
|
|
content_index = 0
|
|
part_type = "audio" if is_audio else "text"
|
|
part_obj = {"type": part_type, "text": "", "audio": "", "transcript": ""}
|
|
|
|
await self._send_event(
|
|
types.ResponseContentPartAddedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.content_part.added",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
part=part_obj, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
|
|
full_text = ""
|
|
full_transcript = ""
|
|
full_token_ids: list[int] = []
|
|
cancelled = False
|
|
limit_reached = False
|
|
usage = ResponseUsage()
|
|
total_audio_samples = 0
|
|
|
|
previous_text = ""
|
|
previous_token_ids: list[int] = []
|
|
pending_tool_calls: dict[int, dict[str, Any]] = {}
|
|
next_output_index = 1 # 0 is the message item, reserved above
|
|
# Qwen's talker speaks tool markup, so stop audio after detecting a call.
|
|
tool_call_seen = False
|
|
|
|
async def emit_content_delta(piece: str) -> None:
|
|
nonlocal full_text, full_transcript
|
|
if not piece:
|
|
return
|
|
full_transcript += piece
|
|
if is_audio:
|
|
await self._send_event(
|
|
types.ResponseAudioTranscriptDeltaEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_audio_transcript.delta",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
delta=piece,
|
|
)
|
|
)
|
|
else:
|
|
full_text += piece
|
|
await self._send_event(
|
|
types.ResponseTextDeltaEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_text.delta",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
delta=piece,
|
|
)
|
|
)
|
|
|
|
async def handle_tool_parser_delta(delta_msg) -> None:
|
|
nonlocal next_output_index, tool_call_seen
|
|
if delta_msg is None:
|
|
return
|
|
if delta_msg.content:
|
|
await emit_content_delta(delta_msg.content)
|
|
for tc in delta_msg.tool_calls:
|
|
entry = pending_tool_calls.get(tc.index)
|
|
if entry is None:
|
|
tool_call_seen = True
|
|
entry = {
|
|
"item_id": _gen_id("item"),
|
|
"call_id": tc.id or _gen_id("call"),
|
|
"name": tc.function.name if tc.function else None,
|
|
"arguments": "",
|
|
"output_index": next_output_index,
|
|
}
|
|
pending_tool_calls[tc.index] = entry
|
|
next_output_index += 1
|
|
await self._send_event(
|
|
types.ResponseOutputItemAddedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_item.added",
|
|
response_id=response_id,
|
|
output_index=entry["output_index"],
|
|
item=types.RealtimeConversationItemFunctionCall(
|
|
type="function_call",
|
|
id=entry["item_id"],
|
|
call_id=entry["call_id"],
|
|
name=entry["name"] or "",
|
|
arguments="",
|
|
status="in_progress",
|
|
), # type: ignore[arg-type]
|
|
)
|
|
)
|
|
elif not entry["name"] and tc.function and tc.function.name:
|
|
entry["name"] = tc.function.name
|
|
|
|
if tc.function and tc.function.arguments:
|
|
entry["arguments"] += tc.function.arguments
|
|
await self._send_event(
|
|
types.ResponseFunctionCallArgumentsDeltaEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.function_call_arguments.delta",
|
|
response_id=response_id,
|
|
item_id=entry["item_id"],
|
|
output_index=entry["output_index"],
|
|
call_id=entry["call_id"],
|
|
delta=tc.function.arguments,
|
|
)
|
|
)
|
|
|
|
# Defaults are process-wide mutable objects.
|
|
sampling_params_list = [
|
|
sp.clone() if isinstance(sp, SamplingParams) else sp for sp in self.engine.default_sampling_params_list
|
|
]
|
|
max_output_tokens = response.max_output_tokens
|
|
thinker_params_configured = False
|
|
for sp in sampling_params_list:
|
|
if isinstance(sp, SamplingParams):
|
|
sp.output_kind = RequestOutputKind.DELTA
|
|
if not thinker_params_configured:
|
|
if isinstance(max_output_tokens, int):
|
|
sp.max_tokens = max_output_tokens
|
|
if structural_tag_json is not None:
|
|
sp.structured_outputs = StructuredOutputsParams(structural_tag=structural_tag_json)
|
|
thinker_params_configured = True
|
|
|
|
gen = self.engine.generate(
|
|
prompt=prompt,
|
|
request_id=active.request_id,
|
|
sampling_params_list=sampling_params_list,
|
|
output_modalities=modalities,
|
|
)
|
|
|
|
try:
|
|
async for output in gen:
|
|
if not self._connected:
|
|
cancelled = True
|
|
break
|
|
|
|
output_type = getattr(output, "final_output_type", "text")
|
|
if output_type == "audio":
|
|
audio_chunks = self._extract_audio_deltas(output)
|
|
for chunk in audio_chunks:
|
|
total_audio_samples += chunk.shape[0]
|
|
# is_audio guard is defense-in-depth: generate() is
|
|
# now given output_modalities=modalities above, so
|
|
# the engine shouldn't produce audio-typed output
|
|
# for a text-only response -- but don't rely on
|
|
# that alone to honor the client's explicit
|
|
# request; never flip session state or forward
|
|
# audio it didn't ask for.
|
|
if not is_audio:
|
|
continue
|
|
if tool_call_seen:
|
|
continue
|
|
b64 = self._pcm16_b64(chunk)
|
|
await self._send_event(
|
|
types.ResponseAudioDeltaEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_audio.delta",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
delta=b64,
|
|
)
|
|
)
|
|
continue
|
|
|
|
if output.outputs:
|
|
first_out = output.outputs[0]
|
|
finish_reason = getattr(first_out, "finish_reason", None)
|
|
finish_reason_value = getattr(finish_reason, "value", finish_reason)
|
|
if finish_reason_value == "length" or str(finish_reason_value).lower() in {
|
|
"finishreason.length",
|
|
"length",
|
|
}:
|
|
limit_reached = True
|
|
delta_text = first_out.text or ""
|
|
delta_token_ids = list(first_out.token_ids)
|
|
usage.output_tokens += len(delta_token_ids)
|
|
# Raw thinker token stream, in talker-consumption order --
|
|
# this is what _qwen3_omni_truncate_transcript correlates
|
|
# against codec frames, independent of any tool-parser
|
|
# stripping applied to full_text/full_transcript below
|
|
# (the talker speaks the raw stream, tool tags included).
|
|
full_token_ids.extend(delta_token_ids)
|
|
|
|
if output.prompt_token_ids:
|
|
usage.input_tokens = max(usage.input_tokens, len(output.prompt_token_ids))
|
|
|
|
if tool_parser is not None:
|
|
# Additive branch: when no tools are configured for
|
|
# this response, tool_parser is None and this whole
|
|
# block is skipped -- the plain-text path below is
|
|
# untouched.
|
|
current_text = previous_text + delta_text
|
|
current_token_ids = previous_token_ids + delta_token_ids
|
|
delta_msg = None
|
|
if delta_text or delta_token_ids:
|
|
delta_msg = tool_parser.extract_tool_calls_streaming(
|
|
previous_text,
|
|
current_text,
|
|
delta_text,
|
|
previous_token_ids,
|
|
current_token_ids,
|
|
delta_token_ids,
|
|
request=parser_request,
|
|
)
|
|
previous_text = current_text
|
|
previous_token_ids = current_token_ids
|
|
await handle_tool_parser_delta(delta_msg)
|
|
elif delta_text:
|
|
await emit_content_delta(delta_text)
|
|
|
|
except asyncio.CancelledError:
|
|
cancelled = True
|
|
finally:
|
|
aclose = getattr(gen, "aclose", None)
|
|
if aclose is not None:
|
|
try:
|
|
await aclose()
|
|
except Exception:
|
|
logger.debug("Error closing generator for %s", active.request_id, exc_info=True)
|
|
|
|
if tool_parser is not None and getattr(tool_parser, "engine_based_streaming", False):
|
|
# finish_streaming() only exists on the newer ParserEngine-based
|
|
# parsers (engine_based_streaming=True, e.g. Qwen3EngineToolParser)
|
|
# -- the base ToolParser class legacy regex-based parsers extend
|
|
# (e.g. Hermes2ProToolParser) don't declare it at all and would
|
|
# raise AttributeError here (confirmed in production logs).
|
|
await handle_tool_parser_delta(tool_parser.finish_streaming())
|
|
|
|
if self._response_cancel_event.is_set():
|
|
cancelled = True
|
|
|
|
status = "cancelled" if cancelled else "incomplete" if limit_reached else "completed"
|
|
|
|
if is_audio:
|
|
await self._send_event(
|
|
types.ResponseAudioDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_audio.done",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
)
|
|
)
|
|
await self._send_event(
|
|
types.ResponseAudioTranscriptDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_audio_transcript.done",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
transcript=full_transcript,
|
|
)
|
|
)
|
|
else:
|
|
await self._send_event(
|
|
types.ResponseTextDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_text.done",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
text=full_text,
|
|
)
|
|
)
|
|
|
|
# Reconstruct rather than mutate item_obj.status/.content in place:
|
|
# pydantic does not validate/coerce plain attribute assignment after
|
|
# construction, so `item_obj.content = [{...}]` would silently leave
|
|
# item_obj.content holding raw dicts instead of Content objects --
|
|
# _assistant_item_text's attribute-based access (part.transcript)
|
|
# then always returns None for those, so no assistant turn's text
|
|
# ever actually made it into a later response's prompt.
|
|
item_obj = types.RealtimeConversationItemAssistantMessage(
|
|
type="message",
|
|
role="assistant",
|
|
id=item_id,
|
|
status="completed" if not cancelled and not limit_reached else "incomplete",
|
|
content=(
|
|
[{"type": "output_audio", "transcript": full_transcript}] # type: ignore[list-item]
|
|
if is_audio
|
|
else [{"type": "output_text", "text": full_text}] # type: ignore[list-item]
|
|
),
|
|
)
|
|
|
|
done_part = {"type": part_type, "text": full_text, "transcript": full_transcript}
|
|
await self._send_event(
|
|
types.ResponseContentPartDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.content_part.done",
|
|
response_id=response_id,
|
|
item_id=item_id,
|
|
output_index=output_index,
|
|
content_index=content_index,
|
|
part=done_part, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
|
|
await self._send_event(
|
|
types.ResponseOutputItemDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_item.done",
|
|
response_id=response_id,
|
|
output_index=output_index,
|
|
item=item_obj, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
|
|
# Per spec, response.done always includes every output item that was
|
|
# generated, regardless of final status -- so the item exists in
|
|
# history either way. For a cancelled response we can't know how
|
|
# much of it the user actually heard (no alignment between audio
|
|
# timing and text/token position -- see conversation.item.truncate),
|
|
# so rather than guess, the history copy gets empty content; the
|
|
# wire events above already carried the real accumulated content.
|
|
history_item = item_obj
|
|
if cancelled:
|
|
history_item = types.RealtimeConversationItemAssistantMessage(
|
|
type="message",
|
|
role="assistant",
|
|
id=item_obj.id,
|
|
status="incomplete",
|
|
content=(
|
|
[{"type": "output_audio", "transcript": ""}] # type: ignore[list-item]
|
|
if is_audio
|
|
else [{"type": "output_text", "text": ""}] # type: ignore[list-item]
|
|
),
|
|
)
|
|
|
|
# A response that only called tools has no message item in `output`
|
|
# (matches real OpenAI behavior) -- drop the placeholder rather than
|
|
# keep an empty message. Scoped to a response that actually
|
|
# completed: a cancelled response keeps today's empty-incomplete
|
|
# message-item behavior regardless of any in-flight tool call, since
|
|
# cancellation-mid-tool-call isn't handled specially here.
|
|
#
|
|
# total_audio_samples == 0 is required, not just empty text: the
|
|
# tool parser classifies raw <tool_call>...</tool_call> text as
|
|
# "not content", but the talker has no concept of that span and
|
|
# synthesizes audio for the whole segment regardless (nothing in
|
|
# _thinker_to_talker_prefill special-cases tool-call text) -- so a
|
|
# "no content" tool-call response can still have real audio that was
|
|
# actually streamed to and played by the client. Dropping the
|
|
# message item in that case orphans that audio: a later
|
|
# conversation.item.truncate against it fails with "not found"
|
|
# (confirmed in production logs, audio_end_ms=6201 on an item that
|
|
# had already been dropped here).
|
|
drop_message_item = (
|
|
not cancelled
|
|
and total_audio_samples == 0
|
|
and not (full_text or full_transcript)
|
|
and bool(pending_tool_calls)
|
|
)
|
|
|
|
chain_after = previous_item_id
|
|
if s.find_item_index(item_id) is not None:
|
|
# Captured before remove_item below, whose own cleanup pops
|
|
# pending_truncations_ms as part of deleting the item -- reading
|
|
# it only after (as the item_in_progress.pop block used to)
|
|
# would silently lose a truncate that arrived while this
|
|
# response was still in progress and later turned out to be
|
|
# tool-call-only (drop_message_item), with no ack or error ever
|
|
# sent back to the client.
|
|
pending_event = s.pending_truncations_ms.get(item_id)
|
|
if drop_message_item:
|
|
s.remove_item(item_id)
|
|
if pending_event is not None:
|
|
await self._send_error(
|
|
f"Item '{item_id}' produced no message content to truncate",
|
|
"invalid_request_error",
|
|
event_id=pending_event.event_id,
|
|
)
|
|
else:
|
|
s.replace_item(history_item)
|
|
if item_obj.id:
|
|
s.item_duration_ms[item_obj.id] = total_audio_samples / SAMPLE_RATE_HZ * 1000
|
|
# Skip storing for tool-call responses: full_token_ids is
|
|
# the raw thinker stream (tool-call tags included), but
|
|
# this item's transcript/text is the tool-parser-stripped
|
|
# content -- the two no longer line up token-for-token,
|
|
# so _qwen3_omni_truncate_transcript falls back to
|
|
# blanking (today's behavior) rather than risk splicing
|
|
# raw <tool_call> text into a truncated transcript.
|
|
if not pending_tool_calls:
|
|
s.item_token_ids[item_obj.id] = full_token_ids
|
|
await self._send_conversation_item_added_and_done(history_item, previous_item_id)
|
|
chain_after = history_item.id
|
|
|
|
# Only now -- after item_duration_ms/item_token_ids are finally
|
|
# populated (or the item is gone, for drop_message_item) -- is
|
|
# it safe to resolve a conversation.item.truncate that arrived
|
|
# while this response was still item_in_progress (see
|
|
# _handle_item_truncate). Clearing item_in_progress first means
|
|
# a truncate arriving from here on goes straight through
|
|
# _handle_item_truncate's normal, non-deferred path.
|
|
s.item_in_progress.pop(item_id, None)
|
|
if not drop_message_item and pending_event is not None:
|
|
await self._do_item_truncate(pending_event)
|
|
|
|
# Function calls aren't subject to the truncate-race protection the
|
|
# message item needed above -- per spec only assistant message items
|
|
# can ever be truncated, so there's no client action that could race
|
|
# ahead and remove one of these before we get here.
|
|
function_call_items: list[types.RealtimeConversationItemFunctionCall] = []
|
|
for entry in pending_tool_calls.values():
|
|
await self._send_event(
|
|
types.ResponseFunctionCallArgumentsDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.function_call_arguments.done",
|
|
response_id=response_id,
|
|
item_id=entry["item_id"],
|
|
output_index=entry["output_index"],
|
|
call_id=entry["call_id"],
|
|
name=entry["name"] or "",
|
|
arguments=entry["arguments"],
|
|
)
|
|
)
|
|
fc_item = types.RealtimeConversationItemFunctionCall(
|
|
type="function_call",
|
|
id=entry["item_id"],
|
|
call_id=entry["call_id"],
|
|
name=entry["name"] or "",
|
|
arguments=entry["arguments"],
|
|
status="completed" if not cancelled and not limit_reached else "incomplete",
|
|
)
|
|
await self._send_event(
|
|
types.ResponseOutputItemDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.output_item.done",
|
|
response_id=response_id,
|
|
output_index=entry["output_index"],
|
|
item=fc_item, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
s.insert_item(fc_item, previous_item_id=chain_after or "root")
|
|
await self._send_conversation_item_added_and_done(fc_item, chain_after)
|
|
chain_after = fc_item.id
|
|
function_call_items.append(fc_item)
|
|
|
|
usage.total_tokens = usage.input_tokens + usage.output_tokens
|
|
|
|
status_details = None
|
|
if cancelled:
|
|
status_details = {
|
|
"type": "cancelled",
|
|
"reason": "client_cancelled",
|
|
}
|
|
elif limit_reached:
|
|
status_details = {
|
|
"type": "incomplete",
|
|
"reason": "max_output_tokens",
|
|
}
|
|
|
|
output_items: list[Any] = [] if drop_message_item else [item_obj]
|
|
output_items.extend(function_call_items)
|
|
|
|
done_response = self._response_object(
|
|
response_id,
|
|
response,
|
|
status,
|
|
status_details=status_details,
|
|
output=output_items,
|
|
usage={
|
|
"total_tokens": usage.total_tokens,
|
|
"input_tokens": usage.input_tokens,
|
|
"output_tokens": usage.output_tokens,
|
|
},
|
|
)
|
|
|
|
if not self._mark_terminal_response(response_id):
|
|
return
|
|
await self._send_event(
|
|
types.ResponseDoneEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="response.done",
|
|
response=done_response,
|
|
)
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# response.cancel #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _handle_response_cancel(self, event: types.ResponseCancelEvent):
|
|
active = self.session.active_response
|
|
if active is None:
|
|
await self._send_error(
|
|
"No response is in progress",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
response_id = getattr(event, "response_id", None)
|
|
if response_id is not None and response_id != active.response_id:
|
|
await self._send_error(
|
|
f"Response '{response_id}' is not in progress",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
await self._cancel_active_response()
|
|
|
|
async def _cancel_active_response(self) -> None:
|
|
active = self.session.active_response
|
|
if active is None:
|
|
return
|
|
self._response_cancel_event.set()
|
|
# engine.abort() tears down orchestrator/stage-pool bookkeeping for
|
|
# the request, but never pushes a completion sentinel through the
|
|
# per-request output queue -- so it does NOT by itself unblock
|
|
# _run_response_inner's `async for output in gen:` loop, which would
|
|
# otherwise wait on that queue forever. Cancelling the task is what
|
|
# actually stops it (injects CancelledError at the current await),
|
|
# which _run_response_inner catches to still emit response.done.
|
|
try:
|
|
await self.engine.abort(active.request_id)
|
|
except Exception:
|
|
logger.exception("Failed to abort request %s", active.request_id)
|
|
if self._response_task and not self._response_task.done():
|
|
self._response_task.cancel()
|
|
try:
|
|
await self._response_task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# conversation.item.create #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _handle_item_create(self, event: types.ConversationItemCreateEvent):
|
|
item = event.item
|
|
|
|
if (getattr(item, "type", None) or "").startswith("mcp_"):
|
|
await self._send_unsupported_mcp(event.event_id)
|
|
return
|
|
|
|
try:
|
|
self._validate_input_item(item)
|
|
s = self.session
|
|
pos = s.insert_item(item, event.previous_item_id)
|
|
except ValueError as e:
|
|
await self._send_error(str(e), "invalid_request_error", event_id=event.event_id)
|
|
return
|
|
prev_id = s.items[pos - 1].id if pos > 0 else None
|
|
|
|
await self._send_conversation_item_added_and_done(item, prev_id)
|
|
|
|
def _validate_input_item(self, item: types.ConversationItem) -> None:
|
|
for part in getattr(item, "content", None) or []:
|
|
part_type = getattr(part, "type", None)
|
|
if part_type == "input_image":
|
|
raise ValueError("Image input is not supported")
|
|
if part_type == "input_audio":
|
|
audio = getattr(part, "audio", None)
|
|
if audio:
|
|
self._decode_pcm16(audio)
|
|
if part_type == "output_audio" and getattr(part, "audio", None):
|
|
raise ValueError("Client-provided assistant audio is not supported")
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# conversation.item.delete #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _handle_item_delete(self, event: types.ConversationItemDeleteEvent):
|
|
item_id = event.item_id
|
|
|
|
# Per spec: "Send this event when you want to remove any item from
|
|
# the conversation history" -- no positional restriction, the only
|
|
# failure case is the item not existing.
|
|
removed = self.session.remove_item(item_id)
|
|
if removed is None:
|
|
await self._send_error(
|
|
f"Item '{item_id}' not found",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
|
|
await self._send_event(
|
|
types.ConversationItemDeletedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="conversation.item.deleted",
|
|
item_id=item_id,
|
|
)
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# conversation.item.retrieve #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _handle_item_retrieve(self, event: types.ConversationItemRetrieveEvent):
|
|
item_id = event.item_id
|
|
|
|
item = self.session.find_item(item_id)
|
|
if item is None:
|
|
await self._send_error(
|
|
f"Item '{item_id}' not found",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
|
|
await self._send_json(
|
|
{
|
|
"event_id": _gen_id("evt"),
|
|
"type": "conversation.item.retrieved",
|
|
"item": item.model_dump(exclude_none=True),
|
|
}
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# conversation.item.truncate #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _handle_item_truncate(self, event: types.ConversationItemTruncateEvent):
|
|
item_id = event.item_id
|
|
s = self.session
|
|
|
|
item = s.find_item(item_id)
|
|
if item is None:
|
|
await self._send_error(
|
|
f"Item '{item_id}' not found",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
|
|
if not isinstance(item, types.RealtimeConversationItemAssistantMessage):
|
|
await self._send_error(
|
|
"Can only truncate assistant messages",
|
|
"invalid_request_error",
|
|
event_id=event.event_id,
|
|
)
|
|
return
|
|
|
|
# Record intent unconditionally. If the item's own response is still
|
|
# generating (item_in_progress), item_duration_ms/item_token_ids
|
|
# don't exist yet and the placeholder's content is still empty --
|
|
# applying the truncation now would be a silent no-op, and
|
|
# _run_response_inner's own finalization write would clobber it
|
|
# afterward regardless (blank-on-cancel, or the full untruncated
|
|
# content if the response isn't cancelled and keeps generating past
|
|
# this point). Deferring to finalization -- the one place
|
|
# item_token_ids is finally complete -- closes that race regardless
|
|
# of which order the two arrive in; see _run_response_inner's own
|
|
# check of this dict at finalization time.
|
|
s.pending_truncations_ms[item_id] = event
|
|
|
|
if not s.item_in_progress.get(item_id, False):
|
|
await self._do_item_truncate(event)
|
|
|
|
async def _do_item_truncate(self, event: types.ConversationItemTruncateEvent) -> None:
|
|
"""Apply a conversation.item.truncate once the target item is no
|
|
longer in progress -- called either directly from
|
|
_handle_item_truncate (item already finalized) or from
|
|
_run_response_inner's finalization (item was still in progress when
|
|
the truncate first arrived)."""
|
|
s = self.session
|
|
item_id = event.item_id
|
|
# A second truncate may have arrived (and overwritten
|
|
# pending_truncations_ms) while the first was still waiting on this
|
|
# same in-progress item -- always honor the latest recorded value
|
|
# rather than the one on the event that happened to trigger this call.
|
|
effective_event = s.pending_truncations_ms.pop(item_id, None) or event
|
|
content_index = effective_event.content_index
|
|
audio_end_ms = effective_event.audio_end_ms
|
|
|
|
item = s.find_item(item_id)
|
|
if item is None:
|
|
# Deleted/removed between the truncate request and its
|
|
# resolution (e.g. conversation.item.delete raced ahead, or
|
|
# drop_message_item removed it in _run_response_inner) --
|
|
# nothing left to truncate.
|
|
return
|
|
if not isinstance(item, types.RealtimeConversationItemAssistantMessage):
|
|
return
|
|
|
|
duration_ms = s.item_duration_ms.get(item_id)
|
|
if duration_ms is not None and audio_end_ms > duration_ms:
|
|
await self._send_error(
|
|
f"audio_end_ms ({audio_end_ms}) is greater than the actual audio duration",
|
|
"invalid_request_error",
|
|
event_id=effective_event.event_id,
|
|
)
|
|
return
|
|
|
|
new_content = list(item.content)
|
|
if not 0 <= content_index < len(new_content):
|
|
# Reject an invalid content index before changing the history.
|
|
await self._send_error(
|
|
f"content_index {content_index} is out of range for item '{item_id}'",
|
|
"invalid_request_error",
|
|
event_id=effective_event.event_id,
|
|
)
|
|
return
|
|
|
|
# Per spec, truncating audio must not leave text in context the user
|
|
# never heard -- but rather than blanking the transcript entirely
|
|
# (the previous, spec-minimum behavior), reconstruct the prefix that
|
|
# *was* actually heard using Qwen3-Omni's fixed talker text/frame
|
|
# correlation (see _qwen3_omni_truncate_transcript). This keeps the
|
|
# model's own memory of what it said in sync with what the user
|
|
# actually heard, instead of wiping it and confusing later turns.
|
|
# Falls back to "" (today's behavior) when we don't have a captured
|
|
# token stream for this item (e.g. it included a tool call).
|
|
truncated_text = self._qwen3_omni_truncate_transcript(item_id, audio_end_ms)
|
|
part = new_content[content_index]
|
|
if hasattr(part, "transcript"):
|
|
new_content[content_index] = {
|
|
"type": "output_audio",
|
|
"transcript": truncated_text,
|
|
}
|
|
elif hasattr(part, "text"):
|
|
new_content[content_index] = {"type": "output_text", "text": truncated_text}
|
|
truncated_item = types.RealtimeConversationItemAssistantMessage(
|
|
type="message",
|
|
role="assistant",
|
|
id=item_id,
|
|
status=item.status,
|
|
content=new_content, # type: ignore[arg-type]
|
|
)
|
|
s.replace_item(truncated_item)
|
|
|
|
await self._send_event(
|
|
types.ConversationItemTruncatedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="conversation.item.truncated",
|
|
item_id=item_id,
|
|
content_index=content_index,
|
|
audio_end_ms=audio_end_ms,
|
|
)
|
|
)
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# History -> prompt serialization #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
@staticmethod
|
|
def _convert_tools(tools: list, *, strict: bool = False) -> list[dict]:
|
|
"""RealtimeFunctionTool is flat (type/name/description/parameters);
|
|
apply_chat_template(tools=...) and ChatCompletionToolsParam both
|
|
expect the chat-completions nested shape. MCP tools
|
|
(RealtimeResponseCreateMcpTool) are not handled -- skipped.
|
|
|
|
strict=True marks every tool as OpenAI chat-completions "strict"
|
|
function calling -- RealtimeFunctionTool has no such field of its
|
|
own, but vLLM's structural-tag builder
|
|
(get_model_structural_tag/_any_tool_strict) only builds a
|
|
tool_choice="auto" grammar when at least one tool is strict, so
|
|
callers building guided-decoding tools need this set. Left False
|
|
for the copy handed to apply_chat_template, which shouldn't show
|
|
clients an OpenAI-specific field their own tool declaration never
|
|
asked for.
|
|
"""
|
|
converted = []
|
|
for tool in tools:
|
|
if getattr(tool, "type", None) != "function":
|
|
continue
|
|
function: dict[str, Any] = {
|
|
"name": tool.name,
|
|
"description": tool.description,
|
|
"parameters": tool.parameters,
|
|
}
|
|
if strict:
|
|
function["strict"] = True
|
|
converted.append({"type": "function", "function": function})
|
|
return converted
|
|
|
|
@staticmethod
|
|
def _convert_tool_choice(
|
|
tool_choice: Any,
|
|
) -> str | ChatCompletionNamedToolChoiceParam:
|
|
if isinstance(tool_choice, str):
|
|
return tool_choice
|
|
if isinstance(tool_choice, dict):
|
|
choice_type = tool_choice.get("type")
|
|
name = tool_choice.get("name")
|
|
else:
|
|
choice_type = getattr(tool_choice, "type", None)
|
|
name = getattr(tool_choice, "name", None)
|
|
if choice_type == "function" and name:
|
|
return ChatCompletionNamedToolChoiceParam(
|
|
type="function",
|
|
function={"name": name},
|
|
)
|
|
return "auto"
|
|
|
|
def _assistant_item_text(self, item: types.RealtimeConversationItemAssistantMessage) -> str:
|
|
"""Return the assistant item's stored transcript or text."""
|
|
for part in item.content:
|
|
text = getattr(part, "transcript", None) or getattr(part, "text", None)
|
|
if text:
|
|
return text
|
|
return ""
|
|
|
|
def _resolve_response_input(self, items: list[Any]) -> list[Any]:
|
|
resolved = []
|
|
for item in items:
|
|
if getattr(item, "type", None) != "item_reference":
|
|
self._validate_input_item(item)
|
|
resolved.append(item)
|
|
continue
|
|
item_id = getattr(item, "id", None)
|
|
referenced = self.session.find_item(item_id) if item_id else None
|
|
if referenced is None:
|
|
raise ValueError(f"Item '{item_id}' not found")
|
|
self._validate_input_item(referenced)
|
|
resolved.append(referenced)
|
|
return resolved
|
|
|
|
def _qwen3_omni_truncate_transcript(self, item_id: str, audio_end_ms: float) -> str:
|
|
"""Approximate the transcript heard before ``audio_end_ms``."""
|
|
token_ids = self.session.item_token_ids.get(item_id)
|
|
if not token_ids:
|
|
return ""
|
|
tokens_heard = round(audio_end_ms / QWEN3_OMNI_MS_PER_TOKEN)
|
|
tokens_heard = min(tokens_heard, len(token_ids))
|
|
if tokens_heard <= 0:
|
|
return ""
|
|
raw_tok = getattr(self._tokenizer, "tokenizer", self._tokenizer)
|
|
return raw_tok.decode(token_ids[:tokens_heard], skip_special_tokens=True)
|
|
|
|
@staticmethod
|
|
def _qwen3_omni_audio_token_count(num_samples: int, sample_rate: int) -> int:
|
|
"""Return Qwen3-Omni's expanded token count for an audio segment."""
|
|
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
|
_get_feat_extract_output_lengths,
|
|
)
|
|
|
|
num_samples_16k = round(num_samples * 16000 / sample_rate)
|
|
input_lengths = num_samples_16k // 160
|
|
output_lengths = _get_feat_extract_output_lengths(input_lengths)
|
|
return int(output_lengths) + 2 # + <|audio_start|> + <|audio_end|>
|
|
|
|
async def _build_full_prompt(
|
|
self,
|
|
tools: list | None = None,
|
|
*,
|
|
instructions: str | None = None,
|
|
items: list | None = None,
|
|
) -> TokensPrompt:
|
|
"""Render the effective instructions and items into an engine prompt."""
|
|
from vllm.inputs import TokensPrompt
|
|
|
|
s = self.session
|
|
effective_instructions = instructions if instructions is not None else s.config.instructions
|
|
effective_items = items if items is not None else s.items
|
|
messages: list[dict[str, Any]] = []
|
|
audio_arrays: list[tuple[np.ndarray, int]] = []
|
|
converted_tools = self._convert_tools(tools) if tools else None
|
|
|
|
if effective_instructions:
|
|
messages.append({"role": "system", "content": effective_instructions})
|
|
|
|
for item in effective_items:
|
|
if item.type == "function_call":
|
|
# "" not None: this model's chat template only handles
|
|
# message.content as a string or a list (line 55/59 of its
|
|
# Jinja template unconditionally iterates non-string content
|
|
# assuming it's a list) -- None crashes with "'NoneType'
|
|
# object is not iterable" (confirmed in production logs).
|
|
messages.append(
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": item.call_id,
|
|
"type": "function",
|
|
"function": {"name": item.name, "arguments": item.arguments},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
continue
|
|
if item.type == "function_call_output":
|
|
messages.append({"role": "tool", "tool_call_id": item.call_id, "content": item.output})
|
|
continue
|
|
|
|
role = getattr(item, "role", None)
|
|
if role == "user":
|
|
parts_text = []
|
|
for part in item.content:
|
|
if part.type == "input_audio" and part.audio:
|
|
parts_text.append(AUDIO_PLACEHOLDER)
|
|
audio_bytes = self._decode_pcm16(part.audio, MAX_INPUT_AUDIO_BUFFER_BYTES)
|
|
pcm16 = np.frombuffer(audio_bytes, dtype=np.int16)
|
|
audio_arrays.append((pcm16.astype(np.float32) / 32768.0, SAMPLE_RATE_HZ))
|
|
elif part.type == "input_text" and part.text:
|
|
parts_text.append(part.text)
|
|
if parts_text:
|
|
messages.append({"role": "user", "content": "".join(parts_text)})
|
|
elif role == "assistant":
|
|
text = self._assistant_item_text(item)
|
|
if text:
|
|
messages.append({"role": "assistant", "content": text})
|
|
elif role == "system":
|
|
text = "".join(part.text for part in item.content if part.type == "input_text" and part.text)
|
|
if text:
|
|
messages.append({"role": "system", "content": text})
|
|
|
|
chat_template_kwargs: dict[str, Any] = {
|
|
"tokenize": False,
|
|
"add_generation_prompt": True,
|
|
"add_special_tokens": False,
|
|
}
|
|
if converted_tools:
|
|
chat_template_kwargs["tools"] = converted_tools
|
|
text = self._tokenizer.apply_chat_template(messages, **chat_template_kwargs)
|
|
raw_tok = getattr(self._tokenizer, "tokenizer", self._tokenizer)
|
|
token_ids = raw_tok.encode(text, add_special_tokens=False)
|
|
|
|
prompt_data = TokensPrompt(prompt_token_ids=token_ids)
|
|
if audio_arrays:
|
|
prompt_data["multi_modal_data"] = {"audio": audio_arrays}
|
|
return prompt_data
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Audio output processing #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
@staticmethod
|
|
def _tensor_to_numpy(value) -> np.ndarray | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, np.ndarray):
|
|
arr = value
|
|
elif hasattr(value, "detach"):
|
|
arr = value.detach().float().cpu().numpy()
|
|
else:
|
|
try:
|
|
arr = np.asarray(value)
|
|
except Exception:
|
|
return None
|
|
if arr.ndim > 1:
|
|
arr = arr.reshape(-1)
|
|
return arr.astype(np.float32, copy=False)
|
|
|
|
def _extract_audio_deltas(self, output) -> list[np.ndarray]:
|
|
"""Return this engine step's new audio samples, as-is.
|
|
|
|
Qwen3-Omni's code2wav already returns non-overlapping increments per
|
|
step -- both chunked_decode and chunked_decode_streaming
|
|
(qwen3_omni_code2wav.py) explicitly slice the left-context overlap
|
|
*out* of their own output before returning it (see
|
|
`wav_chunk[..., context_size * self.total_upsample:]` and the
|
|
`start = left_context_size * total_upsample - tail` slice
|
|
respectively), specifically so nothing downstream has to re-derive
|
|
what's new. This used to also diff each array against a stored
|
|
reference of the previous one (np.allclose with a loose tolerance)
|
|
as a defensive guard -- removed because it was solving a problem
|
|
the model already solves, and had a real failure mode: two
|
|
independent (already-correct) increments landing within tolerance
|
|
of each other by coincidence -- most likely on quiet/near-silent
|
|
passages -- would make it wrongly treat the second one as
|
|
"old prefix + new tail" and silently drop the portion it mistook
|
|
for overlap, corrupting playback for the rest of that response.
|
|
"""
|
|
from collections.abc import Mapping
|
|
|
|
mm = getattr(output, "multimodal_output", None)
|
|
if mm is None or not isinstance(mm, Mapping):
|
|
return []
|
|
|
|
key = "audio" if "audio" in mm else ("model_outputs" if "model_outputs" in mm else None)
|
|
if key is None:
|
|
return []
|
|
|
|
raw_audio = mm.get(key)
|
|
chunks: list[np.ndarray] = []
|
|
|
|
if isinstance(raw_audio, (list, tuple)):
|
|
if raw_audio:
|
|
arr = self._tensor_to_numpy(raw_audio[-1])
|
|
if arr is not None and arr.size > 0:
|
|
chunks.append(arr)
|
|
else:
|
|
arr = self._tensor_to_numpy(raw_audio)
|
|
if arr is not None and arr.size > 0:
|
|
chunks.append(arr)
|
|
return chunks
|
|
|
|
@staticmethod
|
|
def _pcm16_b64(audio_f32: np.ndarray) -> str:
|
|
clipped = np.clip(audio_f32, -1.0, 1.0)
|
|
pcm16 = (clipped * 32767.0).astype(np.int16)
|
|
return base64.b64encode(pcm16.tobytes()).decode()
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Server event emission #
|
|
# ------------------------------------------------------------------ #
|
|
|
|
async def _send_conversation_item_added_and_done(self, item: Any, previous_item_id: str | None) -> None:
|
|
try:
|
|
item_data = self._dump_model(item)
|
|
except Exception:
|
|
logger.exception("[realtime] failed to serialize conversation item")
|
|
self._connected = False
|
|
return
|
|
|
|
for event_type in ("conversation.item.added", "conversation.item.done"):
|
|
await self._send_json(
|
|
{
|
|
"event_id": _gen_id("evt"),
|
|
"type": event_type,
|
|
"previous_item_id": previous_item_id,
|
|
"item": item_data,
|
|
}
|
|
)
|
|
|
|
async def _send_event(self, event) -> None:
|
|
try:
|
|
data = self._dump_model(event) if hasattr(event, "model_dump") else event
|
|
except Exception:
|
|
logger.exception("[realtime] failed to serialize %s", getattr(event, "type", None))
|
|
self._connected = False
|
|
return
|
|
await self._send_payload(data, getattr(event, "type", None))
|
|
|
|
@staticmethod
|
|
def _dump_model(model: Any) -> dict[str, Any]:
|
|
with warnings.catch_warnings():
|
|
warnings.filterwarnings(
|
|
"ignore",
|
|
message="Pydantic serializer warnings",
|
|
category=UserWarning,
|
|
)
|
|
return model.model_dump(mode="json", exclude_none=True)
|
|
|
|
async def _send_json(self, payload: dict) -> None:
|
|
await self._send_payload(payload, payload.get("type"))
|
|
|
|
async def _send_payload(self, payload: Any, event_type: str | None) -> None:
|
|
if not self._connected:
|
|
return
|
|
try:
|
|
async with self._send_lock:
|
|
await self.ws.send_text(json.dumps(payload))
|
|
except Exception:
|
|
logger.warning("[realtime] send failed, marking connection dead: %s", event_type, exc_info=True)
|
|
self._connected = False
|
|
|
|
async def _send_error(
|
|
self,
|
|
message: str,
|
|
code: str = "server_error",
|
|
event_id: str | None = None,
|
|
*,
|
|
error_type: str = "invalid_request_error",
|
|
) -> None:
|
|
error_data: dict[str, Any] = {
|
|
"type": error_type,
|
|
"code": code,
|
|
"message": message,
|
|
"param": None,
|
|
"event_id": event_id,
|
|
}
|
|
await self._send_event(
|
|
types.RealtimeErrorEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="error",
|
|
error=error_data, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
|
|
async def _send_session_created(self) -> None:
|
|
session_obj = self._build_session_object()
|
|
await self._send_event(
|
|
types.SessionCreatedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="session.created",
|
|
session=session_obj, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
|
|
async def _send_session_updated(self) -> None:
|
|
session_obj = self._build_session_object()
|
|
await self._send_event(
|
|
types.SessionUpdatedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="session.updated",
|
|
session=session_obj, # type: ignore[arg-type]
|
|
)
|
|
)
|
|
|
|
async def _send_conversation_created(self) -> None:
|
|
await self._send_event(
|
|
types.ConversationCreatedEvent(
|
|
event_id=_gen_id("evt"),
|
|
type="conversation.created",
|
|
conversation={ # type: ignore[arg-type]
|
|
"id": self.session.conversation_id,
|
|
"object": "realtime.conversation",
|
|
},
|
|
)
|
|
)
|
|
|
|
def _build_session_object(self) -> dict[str, Any]:
|
|
s = self.session
|
|
obj = s.config.model_dump(exclude_none=True)
|
|
obj["object"] = "realtime.session"
|
|
obj["id"] = s.session_id
|
|
obj["expires_at"] = int(s.expires_at)
|
|
return obj
|