246 lines
10 KiB
Python
246 lines
10 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Generic, TypeVar
|
|
|
|
REALTIME_INPUT_AUDIO_FORMATS = {
|
|
"pcm16",
|
|
"pcm_s16le",
|
|
"s16le",
|
|
"pcm_f32le",
|
|
"g711_ulaw",
|
|
"g711_alaw",
|
|
}
|
|
REALTIME_OUTPUT_AUDIO_FORMATS = {
|
|
"pcm16",
|
|
"pcm_s16le",
|
|
"s16le",
|
|
"wav",
|
|
"pcm",
|
|
"g711_ulaw",
|
|
"g711_alaw",
|
|
}
|
|
REALTIME_PCM_DEFAULT_SAMPLE_RATE_HZ = 24_000
|
|
REALTIME_PCM_F32_DEFAULT_SAMPLE_RATE_HZ = 16_000
|
|
REALTIME_G711_DEFAULT_SAMPLE_RATE_HZ = 8_000
|
|
|
|
_RealtimeProtocolConfig = tuple[str, int, str, int | None, float]
|
|
_PendingTurnDetectionUpdate = tuple[
|
|
bool,
|
|
dict[str, object] | None,
|
|
asyncio.Event,
|
|
_RealtimeProtocolConfig,
|
|
]
|
|
|
|
|
|
def realtime_default_sample_rate_hz(fmt: object) -> int | None:
|
|
if not isinstance(fmt, str):
|
|
return None
|
|
normalized = fmt.lower()
|
|
if normalized in {"pcm16", "pcm_s16le", "s16le", "pcm", "audio/pcm"}:
|
|
return REALTIME_PCM_DEFAULT_SAMPLE_RATE_HZ
|
|
if normalized in {"pcm_f32le", "audio/pcm_f32le"}:
|
|
return REALTIME_PCM_F32_DEFAULT_SAMPLE_RATE_HZ
|
|
if normalized in {"g711_ulaw", "g711_alaw", "audio/g711_ulaw", "audio/g711_alaw"}:
|
|
return REALTIME_G711_DEFAULT_SAMPLE_RATE_HZ
|
|
return None
|
|
|
|
|
|
REALTIME_ERROR_TYPES_BY_CODE = {
|
|
"bad_event": "invalid_request_error",
|
|
"bad_audio": "invalid_request_error",
|
|
"config_timeout": "invalid_request_error",
|
|
"invalid_json": "invalid_request_error",
|
|
"event_too_large": "invalid_request_error",
|
|
"unknown_event": "invalid_request_error",
|
|
"internal_error": "server_error",
|
|
"runtime_open_failed": "server_error",
|
|
"runtime_open_unsupported": "server_error",
|
|
"runtime_append_failed": "server_error",
|
|
"runtime_append_task_failed": "server_error",
|
|
"runtime_signal_failed": "server_error",
|
|
"runtime_close_failed": "server_error",
|
|
"runtime_abort_failed": "server_error",
|
|
"runtime_data_plane_stream_failed": "server_error",
|
|
"runtime_data_plane_text_without_audio": "server_error",
|
|
"response_error": "server_error",
|
|
"chat_error": "server_error",
|
|
"server_vad_initialization_failed": "server_error",
|
|
"server_vad_inference_failed": "server_error",
|
|
"duplex_session_busy": "rate_limit_error",
|
|
"resource_exhausted": "rate_limit_error",
|
|
"input_backpressure": "rate_limit_error",
|
|
"response_already_active": "invalid_request_error",
|
|
"response_not_active": "invalid_request_error",
|
|
"response_create_without_input": "invalid_request_error",
|
|
"input_audio_buffer_empty": "invalid_request_error",
|
|
"server_vad_manual_commit_unsupported": "invalid_request_error",
|
|
"missing_item_id": "invalid_request_error",
|
|
"item_not_found": "invalid_request_error",
|
|
"playback_item_mismatch": "invalid_request_error",
|
|
"playback_item_not_found": "invalid_request_error",
|
|
"playback_ack_too_late": "invalid_request_error",
|
|
"unsupported_audio_format": "invalid_request_error",
|
|
"unsupported_ref_audio_path": "invalid_request_error",
|
|
"ref_audio_required": "invalid_request_error",
|
|
"model_update_unsupported": "invalid_request_error",
|
|
"voice_update_after_audio_unsupported": "invalid_request_error",
|
|
"ref_audio_update_unsupported": "invalid_request_error",
|
|
"native_text_append_unsupported": "invalid_request_error",
|
|
"runtime_native_stage_role_required": "server_error",
|
|
"runtime_native_runner_kv_required": "server_error",
|
|
}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _RealtimeResponseState:
|
|
item_id: str
|
|
transcript_parts: list[str] = field(default_factory=list)
|
|
text_parts: list[str] = field(default_factory=list)
|
|
audio_duration_ms: int | None = None
|
|
audio_text_marks: list[dict[str, int]] = field(default_factory=list)
|
|
audio_delta_emitted: bool = False
|
|
audio_done_emitted: bool = False
|
|
audio_part_added: bool = False
|
|
audio_part_done: bool = False
|
|
text_part_added: bool = False
|
|
text_part_done: bool = False
|
|
output_text_done: bool = False
|
|
output_item_done: bool = False
|
|
conversation_item_done: bool = False
|
|
speak_emitted: bool = False
|
|
done_emitted: bool = False
|
|
|
|
@property
|
|
def transcript(self) -> str:
|
|
return "".join(self.transcript_parts)
|
|
|
|
@property
|
|
def text(self) -> str:
|
|
return "".join(self.text_parts)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class RealtimeSessionState:
|
|
"""Single mutable authority for one Realtime protocol session."""
|
|
|
|
_opened: bool = False
|
|
_autostarted_default_session: bool = False
|
|
_resume_only: bool = False
|
|
_pending_outbound: asyncio.Queue[dict[str, object]] = field(default_factory=asyncio.Queue)
|
|
_held_realtime_payloads: list[dict[str, object]] = field(default_factory=list)
|
|
_hold_realtime_output_until_session_created: bool = True
|
|
_default_model: object | None = None
|
|
_default_session_id: object | None = None
|
|
_default_extra_body: dict[str, object] = field(default_factory=dict)
|
|
_input_audio_format: str = "pcm16"
|
|
_input_sample_rate_hz: int = REALTIME_PCM_DEFAULT_SAMPLE_RATE_HZ
|
|
_output_audio_format: str = "pcm16"
|
|
_overlap_silence_rms: float = 0.003
|
|
_turn_detection: dict[str, object] | None = None
|
|
_native_input_append: bool = False
|
|
_pending_turn_detection_update: _PendingTurnDetectionUpdate | None = None
|
|
_send_realtime_json: Any = None
|
|
_initial_session_update: bool = False
|
|
_input_speech_started: bool = False
|
|
_response_states: dict[str | int, _RealtimeResponseState] = field(default_factory=dict)
|
|
_item_truncation_cursors: dict[str, tuple[int, int]] = field(default_factory=dict)
|
|
_active_response_id: str | None = None
|
|
_last_response_id: str | None = None
|
|
_conversation_items: dict[str, dict[str, object]] = field(default_factory=dict)
|
|
_last_conversation_item_id: str | None = None
|
|
_output_sample_rate_hz: int | None = None
|
|
_active_input_item_id: str | None = None
|
|
_input_audio_buffer_has_audio: bool = False
|
|
_input_audio_buffer_had_non_speech: bool = False
|
|
_input_audio_buffer_transcript_parts: list[str] = field(default_factory=list)
|
|
_turn_detection_configured: bool = False
|
|
|
|
@classmethod
|
|
def from_query_params(cls, query_params: Any) -> RealtimeSessionState:
|
|
if hasattr(query_params, "query_params"):
|
|
query_params = query_params.query_params
|
|
getter = query_params.get if hasattr(query_params, "get") else None
|
|
# Canonical query param plus the deprecated model-prefixed alias;
|
|
# both seed the canonical extra_body key.
|
|
native_duplex = None
|
|
if getter is not None:
|
|
native_duplex = getter("native_duplex")
|
|
if native_duplex is None:
|
|
native_duplex = getter("minicpmo45_native_duplex")
|
|
resume_only = getter("resume") if getter is not None else None
|
|
autostart = getter("autostart") if getter is not None else None
|
|
default_extra_body = {}
|
|
if str(native_duplex).strip().lower() in {"1", "true", "yes", "on"}:
|
|
default_extra_body["native_duplex"] = True
|
|
return cls(
|
|
_resume_only=(
|
|
str(resume_only).strip().lower() in {"1", "true", "yes", "on"}
|
|
or str(autostart).strip().lower() in {"0", "false", "no", "off"}
|
|
),
|
|
_default_model=getter("model") if getter is not None else None,
|
|
_default_session_id=(getter("session_id") if getter is not None else None),
|
|
_default_extra_body=default_extra_body,
|
|
)
|
|
|
|
|
|
_StateValue = TypeVar("_StateValue")
|
|
|
|
|
|
class _RealtimeStateField(Generic[_StateValue]):
|
|
"""Explicit descriptor binding one protocol attribute to session state."""
|
|
|
|
def __set_name__(self, owner: type, name: str) -> None:
|
|
del owner
|
|
self._name = name
|
|
|
|
def __get__(self, instance: Any, owner: type | None = None) -> _StateValue | _RealtimeStateField[_StateValue]:
|
|
del owner
|
|
if instance is None:
|
|
return self
|
|
return getattr(instance._state, self._name)
|
|
|
|
def __set__(self, instance: Any, value: _StateValue) -> None:
|
|
setattr(instance._state, self._name, value)
|
|
|
|
|
|
class RealtimeStateOwner:
|
|
"""Declare the mutable state surface shared by input/output projectors."""
|
|
|
|
_state: RealtimeSessionState
|
|
_opened: bool = _RealtimeStateField()
|
|
_autostarted_default_session: bool = _RealtimeStateField()
|
|
_resume_only: bool = _RealtimeStateField()
|
|
_pending_outbound: asyncio.Queue[dict[str, object]] = _RealtimeStateField()
|
|
_held_realtime_payloads: list[dict[str, object]] = _RealtimeStateField()
|
|
_hold_realtime_output_until_session_created: bool = _RealtimeStateField()
|
|
_default_model: object | None = _RealtimeStateField()
|
|
_default_session_id: object | None = _RealtimeStateField()
|
|
_default_extra_body: dict[str, object] = _RealtimeStateField()
|
|
_input_audio_format: str = _RealtimeStateField()
|
|
_input_sample_rate_hz: int = _RealtimeStateField()
|
|
_output_audio_format: str = _RealtimeStateField()
|
|
_overlap_silence_rms: float = _RealtimeStateField()
|
|
_turn_detection: dict[str, object] | None = _RealtimeStateField()
|
|
_native_input_append: bool = _RealtimeStateField()
|
|
_pending_turn_detection_update: _PendingTurnDetectionUpdate | None = _RealtimeStateField()
|
|
_send_realtime_json: Any = _RealtimeStateField()
|
|
_initial_session_update: bool = _RealtimeStateField()
|
|
_input_speech_started: bool = _RealtimeStateField()
|
|
_response_states: dict[str | int, _RealtimeResponseState] = _RealtimeStateField()
|
|
_item_truncation_cursors: dict[str, tuple[int, int]] = _RealtimeStateField()
|
|
_active_response_id: str | None = _RealtimeStateField()
|
|
_last_response_id: str | None = _RealtimeStateField()
|
|
_conversation_items: dict[str, dict[str, object]] = _RealtimeStateField()
|
|
_last_conversation_item_id: str | None = _RealtimeStateField()
|
|
_output_sample_rate_hz: int | None = _RealtimeStateField()
|
|
_active_input_item_id: str | None = _RealtimeStateField()
|
|
_input_audio_buffer_has_audio: bool = _RealtimeStateField()
|
|
_input_audio_buffer_had_non_speech: bool = _RealtimeStateField()
|
|
_input_audio_buffer_transcript_parts: list[str] = _RealtimeStateField()
|
|
_turn_detection_configured: bool = _RealtimeStateField()
|