5d7fdf9350
Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
1490 lines
67 KiB
Python
1490 lines
67 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import binascii
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
|
|
from vllm_omni.entrypoints.duplex.audio import (
|
|
convert_input_audio_with_rate,
|
|
)
|
|
from vllm_omni.entrypoints.duplex.protocol import normalize_native_duplex_key
|
|
from vllm_omni.entrypoints.duplex.realtime_state import (
|
|
REALTIME_INPUT_AUDIO_FORMATS,
|
|
REALTIME_OUTPUT_AUDIO_FORMATS,
|
|
RealtimeStateOwner,
|
|
)
|
|
from vllm_omni.entrypoints.duplex.vad import (
|
|
SILERO_VAD_MIN_THRESHOLD,
|
|
ServerVADUnavailableError,
|
|
SileroStreamingVAD,
|
|
SileroVADConfig,
|
|
StreamingVADResult,
|
|
)
|
|
|
|
|
|
class RealtimeInputTranslator(RealtimeStateOwner):
|
|
"""Translate and validate client Realtime events for the duplex core."""
|
|
|
|
_opened: bool
|
|
_initial_session_update: bool
|
|
_input_speech_started: bool
|
|
_input_audio_buffer_has_audio: bool
|
|
_input_audio_buffer_had_non_speech: bool
|
|
_input_audio_buffer_transcript_parts: list[str]
|
|
_active_input_item_id: str | None
|
|
_last_conversation_item_id: str | None
|
|
_active_response_id: str | None
|
|
_last_response_id: str | None
|
|
_input_audio_format: str
|
|
_output_audio_format: str
|
|
_input_sample_rate_hz: int
|
|
_conversation_items: dict[str, dict[str, object]]
|
|
_item_truncation_cursors: dict[str, tuple[int, int]]
|
|
_pending_outbound: asyncio.Queue[dict[str, object]]
|
|
_pending_commit_item_ids: asyncio.Queue[str]
|
|
|
|
async def _send_realtime_payload(self, payload: dict[str, object]) -> None:
|
|
raise NotImplementedError
|
|
|
|
def _realtime_error_payload(
|
|
self,
|
|
code: str,
|
|
message: str,
|
|
*,
|
|
event_id: object | None = None,
|
|
param: object | None = None,
|
|
) -> dict[str, object]:
|
|
raise NotImplementedError
|
|
|
|
_response_is_done: Callable[[object], bool]
|
|
|
|
async def discard_pending_input_audio(
|
|
self,
|
|
*,
|
|
audio_end_ms: int | None = None,
|
|
) -> None:
|
|
"""Drop Realtime input-buffer state that was consumed as overlap.
|
|
|
|
The serving overlap policy may classify a short acknowledgement such
|
|
as "continue" as a side-channel signal instead of user turn input. The
|
|
Realtime protocol has already observed the append and may have emitted
|
|
``speech_started``; reset that transient buffer so a later commit does
|
|
not accidentally attach the acknowledgement transcript/audio to the
|
|
next real user turn.
|
|
"""
|
|
if self._input_speech_started and self._active_input_item_id is not None:
|
|
await self._send_realtime_payload(
|
|
{
|
|
"type": "input_audio_buffer.speech_stopped",
|
|
"audio_end_ms": max(0, int(audio_end_ms or 0)),
|
|
"item_id": self._active_input_item_id,
|
|
}
|
|
)
|
|
self._input_speech_started = False
|
|
self._active_input_item_id = None
|
|
self._input_audio_buffer_has_audio = False
|
|
self._input_audio_buffer_had_non_speech = False
|
|
self._input_audio_buffer_transcript_parts.clear()
|
|
|
|
async def _send_realtime_input_ack(self, event: dict[str, object]) -> None:
|
|
if event.get("type") != "conversation.item.create":
|
|
return
|
|
item = event.get("item")
|
|
if not isinstance(item, dict):
|
|
return
|
|
item = self._normalize_conversation_item(item)
|
|
previous_item_id = event.get("previous_item_id")
|
|
if isinstance(previous_item_id, str):
|
|
item["_previous_item_id"] = previous_item_id
|
|
if item.get("role") != "user":
|
|
return
|
|
self._conversation_items[str(item["id"])] = item
|
|
for payload in self._conversation_item_added_events(item):
|
|
await self._send_realtime_payload(payload)
|
|
|
|
async def _to_duplex_event(self, event: dict[str, object]) -> dict[str, object] | None:
|
|
event_type = event.get("type")
|
|
if event_type == "session.update":
|
|
session = event.get("session")
|
|
session_payload: dict[str, object] = session if isinstance(session, dict) else event
|
|
format_error = self._validate_realtime_session_audio_formats(session_payload)
|
|
if format_error is not None:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"unsupported_audio_format",
|
|
format_error,
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
turn_detection_error = self._validate_realtime_turn_detection(session_payload)
|
|
if turn_detection_error is not None:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"unsupported_turn_detection",
|
|
turn_detection_error,
|
|
event_id=event.get("event_id"),
|
|
param="turn_detection",
|
|
)
|
|
)
|
|
return None
|
|
self._prepare_realtime_turn_detection_update(session_payload)
|
|
self._apply_realtime_session_defaults(session_payload)
|
|
session_payload.update(self._realtime_overlap_fields(session_payload))
|
|
if not self._opened:
|
|
self._opened = True
|
|
self._initial_session_update = True
|
|
return self._session_create_from_realtime(session_payload)
|
|
return {
|
|
"type": "turn.signal",
|
|
"event": "session.update",
|
|
"payload": session_payload,
|
|
"realtime_event_id": event.get("event_id"),
|
|
}
|
|
if event_type == "conversation.item.create":
|
|
item = event.get("item")
|
|
format_error = self._validate_conversation_item_audio_formats(item)
|
|
if format_error is not None:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"unsupported_audio_format",
|
|
format_error,
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
return await self._conversation_item_to_duplex(event)
|
|
if event_type == "conversation.item.delete":
|
|
item_id = event.get("item_id")
|
|
if not isinstance(item_id, str) or not item_id:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"missing_item_id",
|
|
"conversation.item.delete requires item_id",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
if item_id not in self._conversation_items:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"item_not_found",
|
|
f"Conversation item not found: {item_id}",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
self._remove_conversation_item(item_id)
|
|
await self._pending_outbound.put(
|
|
{
|
|
"type": "turn.signal",
|
|
"event": "conversation.item.delete",
|
|
"payload": {"item_id": item_id},
|
|
}
|
|
)
|
|
return None
|
|
if event_type == "conversation.item.retrieve":
|
|
item_id = event.get("item_id")
|
|
if not isinstance(item_id, str) or not item_id:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"missing_item_id",
|
|
"conversation.item.retrieve requires item_id",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
item = self._conversation_items.get(item_id)
|
|
if item is None:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"item_not_found",
|
|
f"Conversation item not found: {item_id}",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
await self._send_realtime_payload({"type": "conversation.item.retrieved", "item": item})
|
|
return None
|
|
if event_type == "conversation.item.truncate":
|
|
item_id = event.get("item_id")
|
|
audio_end_ms = event.get("audio_end_ms")
|
|
content_index = event.get("content_index", 0)
|
|
if not isinstance(item_id, str) or not item_id:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"missing_item_id",
|
|
"conversation.item.truncate requires item_id",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
if not isinstance(audio_end_ms, int | float):
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"bad_event",
|
|
"conversation.item.truncate requires numeric audio_end_ms",
|
|
event_id=event.get("event_id"),
|
|
param="audio_end_ms",
|
|
)
|
|
)
|
|
return None
|
|
item = self._conversation_items.get(item_id)
|
|
if not isinstance(item, dict):
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"item_not_found",
|
|
f"Conversation item not found: {item_id}",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
truncate_error = self._validate_realtime_item_truncate(
|
|
item,
|
|
content_index=int(content_index) if isinstance(content_index, int | float) else 0,
|
|
audio_end_ms=int(audio_end_ms),
|
|
)
|
|
if truncate_error is not None:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"bad_event",
|
|
truncate_error,
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
self._item_truncation_cursors[item_id] = (
|
|
int(content_index) if isinstance(content_index, int | float) else 0,
|
|
int(audio_end_ms),
|
|
)
|
|
self._truncate_realtime_item_content(
|
|
item,
|
|
content_index=int(content_index) if isinstance(content_index, int | float) else 0,
|
|
audio_end_ms=int(audio_end_ms),
|
|
)
|
|
await self._pending_outbound.put(
|
|
{
|
|
"type": "turn.signal",
|
|
"event": "conversation.item.truncate",
|
|
"payload": {
|
|
"item_id": item_id,
|
|
"content_index": content_index,
|
|
"audio_end_ms": audio_end_ms,
|
|
},
|
|
}
|
|
)
|
|
await self._pending_outbound.put(
|
|
{
|
|
"type": "playback.ack",
|
|
"item_id": item_id,
|
|
"committed_ms": int(audio_end_ms),
|
|
"played_ms": int(audio_end_ms),
|
|
"truncate": True,
|
|
}
|
|
)
|
|
return None
|
|
if event_type == "input_audio_buffer.append":
|
|
audio = event.get("audio") or event.get("delta")
|
|
fmt, format_rate = self._parse_realtime_audio_format(
|
|
event.get("format") or event.get("input_audio_format") or self._input_audio_format
|
|
)
|
|
sample_rate_hz = (
|
|
event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or self._input_sample_rate_hz
|
|
)
|
|
if not self._is_supported_realtime_input_format(fmt):
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"unsupported_audio_format",
|
|
f"Unsupported input_audio_format: {fmt}",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
try:
|
|
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
|
|
audio,
|
|
fmt,
|
|
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
|
|
)
|
|
except ValueError as exc:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"bad_event",
|
|
str(exc),
|
|
event_id=event.get("event_id"),
|
|
param="sample_rate_hz",
|
|
)
|
|
)
|
|
return None
|
|
detector = self._server_vad
|
|
vad_result: StreamingVADResult | None = None
|
|
if isinstance(detector, SileroStreamingVAD):
|
|
try:
|
|
vad_result = detector.process_base64(audio, fmt=fmt, sample_rate_hz=sample_rate_hz)
|
|
except (ServerVADUnavailableError, ValueError) as exc:
|
|
unavailable = isinstance(exc, ServerVADUnavailableError)
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"server_vad_unavailable" if unavailable else "bad_audio",
|
|
str(exc),
|
|
event_id=event.get("event_id"),
|
|
param="turn_detection" if unavailable else "audio",
|
|
)
|
|
)
|
|
return None
|
|
looks_like_speech = (
|
|
vad_result.is_speech
|
|
if vad_result is not None
|
|
else self._input_looks_like_speech(event, audio=audio, fmt=fmt)
|
|
)
|
|
self._input_audio_buffer_has_audio = self._input_audio_buffer_has_audio or (
|
|
looks_like_speech and isinstance(audio, str) and bool(audio)
|
|
)
|
|
self._input_audio_buffer_had_non_speech = self._input_audio_buffer_had_non_speech or (
|
|
not looks_like_speech and isinstance(audio, str) and bool(audio)
|
|
)
|
|
stopped_event: dict[str, object] | None = None
|
|
if vad_result is not None and vad_result.speech_stopped:
|
|
stopped_event = dict(event)
|
|
if vad_result.speech_end_ms is not None:
|
|
stopped_event["audio_end_ms"] = vad_result.speech_end_ms
|
|
if stopped_event is not None and vad_result is not None and vad_result.speech_active:
|
|
await self._emit_input_speech_stopped(
|
|
stopped_event,
|
|
item_id=self._active_input_item_id or f"item_{uuid4().hex}",
|
|
)
|
|
if vad_result is not None and vad_result.speech_started:
|
|
started_event = dict(event)
|
|
if vad_result.speech_start_ms is not None:
|
|
started_event["audio_start_ms"] = vad_result.speech_start_ms
|
|
await self._emit_input_speech_started(started_event)
|
|
elif vad_result is None and looks_like_speech:
|
|
await self._emit_input_speech_started(event)
|
|
if stopped_event is not None and vad_result is not None and not vad_result.speech_active:
|
|
await self._emit_input_speech_stopped(
|
|
stopped_event,
|
|
item_id=self._active_input_item_id or f"item_{uuid4().hex}",
|
|
)
|
|
if looks_like_speech:
|
|
self._remember_input_transcript_hint(event)
|
|
payload = {
|
|
"type": "input_audio_buffer.append",
|
|
"audio": audio,
|
|
"format": fmt,
|
|
"sample_rate_hz": sample_rate_hz,
|
|
}
|
|
video_frames = event.get("video_frames")
|
|
if video_frames is not None:
|
|
frames_error = self._validate_realtime_video_frames(video_frames, event.get("max_slice_nums"))
|
|
if frames_error is not None:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"invalid_video_frames",
|
|
frames_error,
|
|
event_id=event.get("event_id"),
|
|
param="video_frames",
|
|
)
|
|
)
|
|
return None
|
|
if isinstance(video_frames, list):
|
|
payload["video_frames"] = [frame for frame in video_frames if isinstance(frame, str) and frame]
|
|
self._copy_realtime_input_hints(event, payload)
|
|
payload["is_speech"] = looks_like_speech
|
|
if vad_result is not None:
|
|
payload["vad"] = {
|
|
"backend": "silero",
|
|
"is_speech": vad_result.is_speech,
|
|
"speech_active": vad_result.speech_active,
|
|
"speech_started": vad_result.speech_started,
|
|
"speech_stopped": vad_result.speech_stopped,
|
|
"speech_probability": vad_result.speech_probability,
|
|
}
|
|
if vad_result.speech_active:
|
|
payload["force_listen"] = True
|
|
return payload
|
|
if event_type == "input_audio_buffer.commit":
|
|
if not self._input_audio_buffer_has_audio:
|
|
if self._input_audio_buffer_had_non_speech:
|
|
self._input_audio_buffer_had_non_speech = False
|
|
self._active_input_item_id = None
|
|
self._reset_server_vad()
|
|
return {
|
|
"type": "input_audio_buffer.commit",
|
|
"final": event.get("final", True),
|
|
"response_create": False,
|
|
"is_speech": False,
|
|
}
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"input_audio_buffer_empty",
|
|
"input_audio_buffer.commit requires a non-empty input audio buffer",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
item_id = self._active_input_item_id or f"item_{uuid4().hex}"
|
|
await self._pending_commit_item_ids.put(item_id)
|
|
await self._emit_input_speech_stopped(event, item_id=item_id)
|
|
transcript = self._consume_input_transcript_hint()
|
|
self._active_input_item_id = None
|
|
self._input_audio_buffer_has_audio = False
|
|
self._input_audio_buffer_had_non_speech = False
|
|
self._reset_server_vad()
|
|
payload = {
|
|
"type": "input_audio_buffer.commit",
|
|
"final": event.get("final", True),
|
|
"realtime_item_id": item_id,
|
|
"response_create": bool(event.get("response_create", False)),
|
|
}
|
|
if transcript:
|
|
payload["transcript"] = transcript
|
|
return payload
|
|
if event_type == "input_audio_buffer.clear":
|
|
self._input_speech_started = False
|
|
self._active_input_item_id = None
|
|
self._input_audio_buffer_has_audio = False
|
|
self._input_audio_buffer_had_non_speech = False
|
|
self._input_audio_buffer_transcript_parts.clear()
|
|
self._reset_server_vad()
|
|
return {"type": "input_audio_buffer.clear", "reason": event_type}
|
|
if event_type == "output_audio_buffer.clear":
|
|
payload = {"type": "output_audio_buffer.clear", "reason": event_type}
|
|
response_id = event.get("response_id")
|
|
if not isinstance(response_id, str) or not response_id:
|
|
response_id = self._active_response_id or self._last_response_id
|
|
if self._response_is_done(response_id):
|
|
await self._send_realtime_payload(
|
|
{
|
|
"type": "output_audio_buffer.cleared",
|
|
"response_id": response_id,
|
|
}
|
|
)
|
|
return None
|
|
if isinstance(response_id, str) and response_id:
|
|
payload["response_id"] = response_id
|
|
return payload
|
|
if event_type == "response.cancel":
|
|
payload = {"type": "response.cancel", "reason": event_type}
|
|
response_id = event.get("response_id")
|
|
if not isinstance(response_id, str) or not response_id:
|
|
response_id = self._active_response_id or self._last_response_id
|
|
if self._response_is_done(response_id):
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"response_not_active",
|
|
f"Response is already complete: {response_id}",
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
if isinstance(response_id, str) and response_id:
|
|
payload["response_id"] = response_id
|
|
return payload
|
|
if event_type == "response.create":
|
|
response_payload = event.get("response")
|
|
if isinstance(response_payload, dict):
|
|
format_error = self._validate_realtime_response_audio_formats(response_payload)
|
|
if format_error is not None:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"unsupported_audio_format",
|
|
format_error,
|
|
event_id=event.get("event_id"),
|
|
)
|
|
)
|
|
return None
|
|
return {
|
|
"type": "response.create",
|
|
"response": response_payload if isinstance(response_payload, dict) else {},
|
|
}
|
|
if event_type in {"session.close", "close"}:
|
|
return {"type": "session.close"}
|
|
return event
|
|
|
|
@staticmethod
|
|
def _normalize_conversation_item(item: dict[str, object]) -> dict[str, object]:
|
|
normalized = dict(item)
|
|
normalized.setdefault("id", f"item_{uuid4().hex}")
|
|
normalized.setdefault("object", "realtime.item")
|
|
normalized.setdefault("type", "message")
|
|
normalized.setdefault("status", "completed")
|
|
if "role" not in normalized and normalized.get("type") == "message":
|
|
normalized["role"] = "user"
|
|
if not isinstance(normalized.get("content"), list):
|
|
normalized["content"] = []
|
|
return normalized
|
|
|
|
@staticmethod
|
|
def _truncate_realtime_item_content(
|
|
item: dict[str, object],
|
|
*,
|
|
content_index: int,
|
|
audio_end_ms: int,
|
|
) -> None:
|
|
content = item.get("content")
|
|
if not isinstance(content, list) or not content:
|
|
return
|
|
index = max(0, int(content_index))
|
|
if index >= len(content):
|
|
return
|
|
part = content[index]
|
|
if not isinstance(part, dict):
|
|
return
|
|
transcript = part.get("transcript")
|
|
if not isinstance(transcript, str) or not transcript:
|
|
return
|
|
marks = part.get("audio_text_marks")
|
|
if isinstance(marks, list):
|
|
keep_chars = RealtimeInputTranslator._text_chars_for_audio_ms_from_marks(
|
|
audio_end_ms,
|
|
len(transcript),
|
|
marks,
|
|
final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"),
|
|
)
|
|
part["transcript"] = transcript[:keep_chars].rstrip()
|
|
return
|
|
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
|
if isinstance(duration_ms, int | float) and duration_ms > 0:
|
|
keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms))))
|
|
part["transcript"] = transcript[:keep_chars].rstrip()
|
|
elif audio_end_ms <= 0:
|
|
part["transcript"] = ""
|
|
|
|
@staticmethod
|
|
def _validate_realtime_item_truncate(
|
|
item: dict[str, object],
|
|
*,
|
|
content_index: int,
|
|
audio_end_ms: int,
|
|
) -> str | None:
|
|
if item.get("type") != "message" or item.get("role") != "assistant":
|
|
return "conversation.item.truncate only supports assistant message items"
|
|
if audio_end_ms < 0:
|
|
return "conversation.item.truncate requires non-negative audio_end_ms"
|
|
content = item.get("content")
|
|
if not isinstance(content, list) or not content:
|
|
return None
|
|
index = max(0, int(content_index))
|
|
if index >= len(content):
|
|
return f"conversation.item.truncate content_index out of range: {content_index}"
|
|
part = content[index]
|
|
if not isinstance(part, dict):
|
|
return "conversation.item.truncate target content part is invalid"
|
|
if part.get("type") not in {"audio", "output_audio"}:
|
|
return "conversation.item.truncate target content part must be audio"
|
|
duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms")
|
|
if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms):
|
|
return "conversation.item.truncate audio_end_ms exceeds item audio duration"
|
|
return None
|
|
|
|
@staticmethod
|
|
def _text_chars_for_audio_ms_from_marks(
|
|
audio_end_ms: int,
|
|
text_len: int,
|
|
marks: list[object],
|
|
*,
|
|
final_ms: object | None = None,
|
|
) -> int:
|
|
if text_len <= 0:
|
|
return 0
|
|
clean_marks: list[tuple[int, int]] = []
|
|
for mark in marks:
|
|
if not isinstance(mark, dict):
|
|
continue
|
|
raw_text_chars = mark.get("text_chars")
|
|
raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms"))
|
|
if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
|
|
continue
|
|
clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars)))))
|
|
if not clean_marks:
|
|
return 0 if audio_end_ms <= 0 else text_len
|
|
clean_marks.sort(key=lambda item: item[0])
|
|
audio_end_ms = max(0, int(audio_end_ms))
|
|
if audio_end_ms <= 0:
|
|
return 0
|
|
previous_ms = 0
|
|
previous_chars = 0
|
|
for mark_ms, mark_chars in clean_marks:
|
|
mark_ms = max(previous_ms, mark_ms)
|
|
mark_chars = max(previous_chars, min(text_len, mark_chars))
|
|
if audio_end_ms <= mark_ms:
|
|
if mark_ms <= previous_ms:
|
|
return mark_chars
|
|
ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms)
|
|
return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio)))
|
|
previous_ms = mark_ms
|
|
previous_chars = mark_chars
|
|
if isinstance(final_ms, int | float) and int(final_ms) > previous_ms:
|
|
if audio_end_ms >= int(final_ms):
|
|
return text_len
|
|
ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms)
|
|
return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio)))
|
|
return text_len if audio_end_ms >= previous_ms else previous_chars
|
|
|
|
def _conversation_item_added_events(self, item: dict[str, object]) -> list[dict[str, object]]:
|
|
item_id = item.get("id")
|
|
explicit_previous_item_id = item.pop("_previous_item_id", None)
|
|
previous_item_id = (
|
|
explicit_previous_item_id if isinstance(explicit_previous_item_id, str) else self._last_conversation_item_id
|
|
)
|
|
if isinstance(item_id, str) and item_id:
|
|
self._last_conversation_item_id = item_id
|
|
return [
|
|
{
|
|
"type": "conversation.item.added",
|
|
"previous_item_id": previous_item_id,
|
|
"item": item,
|
|
},
|
|
{
|
|
"type": "conversation.item.created",
|
|
"previous_item_id": previous_item_id,
|
|
"item": item,
|
|
},
|
|
]
|
|
|
|
def _conversation_item_done_event(self, item: dict[str, object]) -> dict[str, object]:
|
|
item_id = item.get("id")
|
|
previous_item_id = self._previous_item_id(item_id) if isinstance(item_id, str) else None
|
|
if isinstance(item_id, str) and item_id:
|
|
self._conversation_items[item_id] = item
|
|
self._last_conversation_item_id = item_id
|
|
return {
|
|
"type": "conversation.item.done",
|
|
"previous_item_id": previous_item_id,
|
|
"item": item,
|
|
}
|
|
|
|
def _remove_conversation_item(self, item_id: str) -> bool:
|
|
removed = self._conversation_items.pop(item_id, None) is not None
|
|
if self._last_conversation_item_id == item_id:
|
|
remaining_ids = list(self._conversation_items)
|
|
self._last_conversation_item_id = remaining_ids[-1] if remaining_ids else None
|
|
self._item_truncation_cursors.pop(item_id, None)
|
|
return removed
|
|
|
|
def _response_output_item_added_events(
|
|
self,
|
|
*,
|
|
response_id: object,
|
|
item: dict[str, object],
|
|
) -> list[dict[str, object]]:
|
|
return [
|
|
{
|
|
"type": "response.output_item.added",
|
|
"response_id": response_id,
|
|
"output_index": 0,
|
|
"item": item,
|
|
},
|
|
]
|
|
|
|
@staticmethod
|
|
def _response_content_part(*, transcript: str = "") -> dict[str, object]:
|
|
return {
|
|
"type": "audio",
|
|
"transcript": transcript,
|
|
}
|
|
|
|
@staticmethod
|
|
def _response_text_content_part(*, text: str = "") -> dict[str, object]:
|
|
return {
|
|
"type": "text",
|
|
"text": text,
|
|
}
|
|
|
|
@staticmethod
|
|
def _response_item_content_part(
|
|
*,
|
|
transcript: str = "",
|
|
audio_duration_ms: int | None = None,
|
|
audio_text_marks: list[dict[str, int]] | None = None,
|
|
) -> dict[str, object]:
|
|
part: dict[str, object] = {
|
|
"type": "output_audio",
|
|
"transcript": transcript,
|
|
}
|
|
if audio_duration_ms is not None:
|
|
part["audio_duration_ms"] = int(audio_duration_ms)
|
|
if audio_text_marks:
|
|
part["audio_text_marks"] = [dict(mark) for mark in audio_text_marks]
|
|
return part
|
|
|
|
@staticmethod
|
|
def _response_item_text_content_part(*, text: str = "") -> dict[str, object]:
|
|
return {
|
|
"type": "output_text",
|
|
"text": text,
|
|
}
|
|
|
|
def _previous_item_id(self, item_id: str) -> str | None:
|
|
previous: str | None = None
|
|
for known_id in self._conversation_items:
|
|
if known_id == item_id:
|
|
return previous
|
|
previous = known_id
|
|
if self._last_conversation_item_id == item_id:
|
|
return previous
|
|
return self._last_conversation_item_id
|
|
|
|
def _input_audio_buffer_committed_event(
|
|
self,
|
|
*,
|
|
item_id: str,
|
|
event: dict[str, Any],
|
|
) -> dict[str, object]:
|
|
return {
|
|
"type": "input_audio_buffer.committed",
|
|
"previous_item_id": self._previous_item_id(item_id),
|
|
"item_id": item_id,
|
|
"event": event,
|
|
}
|
|
|
|
def _session_create_from_realtime(self, session_payload: dict[str, object]) -> dict[str, object]:
|
|
self._apply_realtime_session_defaults(session_payload)
|
|
model = session_payload.get("model")
|
|
audio_config = session_payload.get("audio")
|
|
audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None
|
|
audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None
|
|
extra_body_payload = session_payload.get("extra_body")
|
|
extra_body = (
|
|
normalize_native_duplex_key(dict(extra_body_payload)) if isinstance(extra_body_payload, dict) else {}
|
|
)
|
|
extra_body["realtime_session_payload"] = self._json_safe_realtime_payload(session_payload)
|
|
if isinstance(session_payload.get("tools"), list):
|
|
extra_body["realtime_tools"] = session_payload["tools"]
|
|
if isinstance(session_payload.get("tool_choice"), str | dict):
|
|
extra_body["realtime_tool_choice"] = session_payload["tool_choice"]
|
|
metadata = session_payload.get("metadata")
|
|
if isinstance(metadata, dict):
|
|
extra_body["realtime_metadata"] = dict(metadata)
|
|
include = session_payload.get("include")
|
|
if isinstance(include, list):
|
|
extra_body["realtime_include"] = list(include)
|
|
prompt = session_payload.get("prompt")
|
|
if isinstance(prompt, dict):
|
|
extra_body["realtime_prompt"] = dict(prompt)
|
|
input_audio_transcription = self._input_audio_transcription_config(session_payload)
|
|
if isinstance(input_audio_transcription, dict):
|
|
extra_body["realtime_input_audio_transcription"] = dict(input_audio_transcription)
|
|
noise_reduction = session_payload.get("input_audio_noise_reduction")
|
|
if isinstance(noise_reduction, dict):
|
|
extra_body["realtime_input_audio_noise_reduction"] = dict(noise_reduction)
|
|
audio_noise_reduction = audio_input.get("noise_reduction") if isinstance(audio_input, dict) else None
|
|
if isinstance(audio_noise_reduction, dict):
|
|
extra_body["realtime_input_audio_noise_reduction"] = dict(audio_noise_reduction)
|
|
audio = session_payload.get("audio")
|
|
if isinstance(audio, dict):
|
|
extra_body["realtime_audio"] = dict(audio)
|
|
if isinstance(session_payload.get("tracing"), str | dict):
|
|
extra_body["realtime_tracing"] = session_payload["tracing"]
|
|
response_format = self._duplex_response_format(self._output_audio_format)
|
|
extra_body.setdefault("realtime_output_audio_format", self._output_audio_format)
|
|
overlap_fields = self._realtime_overlap_fields(session_payload)
|
|
voice = session_payload.get("voice")
|
|
if not isinstance(voice, str) and isinstance(audio_output, dict):
|
|
voice = audio_output.get("voice")
|
|
speed = session_payload.get("speed")
|
|
if not isinstance(speed, int | float) and isinstance(audio_output, dict):
|
|
speed = audio_output.get("speed")
|
|
return {
|
|
"type": "session.create",
|
|
"session_id": session_payload.get("session_id") or session_payload.get("id"),
|
|
"session": {
|
|
"model": model,
|
|
"modalities": (
|
|
session_payload.get("modalities") or session_payload.get("output_modalities") or ["text", "audio"]
|
|
),
|
|
"instructions": session_payload.get("instructions"),
|
|
"voice": voice,
|
|
"ref_audio": session_payload.get("ref_audio"),
|
|
"response_format": response_format,
|
|
"temperature": session_payload.get("temperature"),
|
|
"max_tokens": self.realtime_max_output_tokens(
|
|
session_payload.get("max_response_output_tokens")
|
|
or session_payload.get("max_output_tokens")
|
|
or session_payload.get("max_tokens")
|
|
),
|
|
"speed": speed,
|
|
"idle_timeout_s": session_payload.get("idle_timeout_s") or 300.0,
|
|
**overlap_fields,
|
|
"extra_body": extra_body,
|
|
},
|
|
}
|
|
|
|
def _apply_realtime_session_defaults(self, session_payload: dict[str, object]) -> None:
|
|
input_format: object = session_payload.get("input_audio_format")
|
|
audio_config = session_payload.get("audio")
|
|
if input_format is None and isinstance(audio_config, dict):
|
|
audio_input = audio_config.get("input")
|
|
if isinstance(audio_input, dict):
|
|
input_format = audio_input.get("format")
|
|
input_format, input_rate = self._parse_realtime_audio_format(input_format)
|
|
if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS:
|
|
self._input_audio_format = input_format
|
|
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
|
|
output_rate_raw: object | None = None
|
|
if output_format is None and isinstance(audio_config, dict):
|
|
audio_output = audio_config.get("output")
|
|
if isinstance(audio_output, dict):
|
|
output_format = audio_output.get("format")
|
|
output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate")
|
|
output_format, output_rate = self._parse_realtime_audio_format(output_format)
|
|
if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0:
|
|
output_rate = int(output_rate_raw)
|
|
if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS:
|
|
self._output_audio_format = self._realtime_output_format(output_format)
|
|
sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate")
|
|
if sample_rate is None and isinstance(audio_config, dict):
|
|
audio_input = audio_config.get("input")
|
|
if isinstance(audio_input, dict):
|
|
sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate")
|
|
if sample_rate is None:
|
|
sample_rate = input_rate
|
|
if isinstance(sample_rate, int | float) and sample_rate > 0:
|
|
self._input_sample_rate_hz = int(sample_rate)
|
|
if isinstance(output_rate, int | float) and output_rate > 0:
|
|
self._output_sample_rate_hz = int(output_rate)
|
|
overlap_fields = self._realtime_overlap_fields(session_payload)
|
|
overlap_silence_rms = overlap_fields.get("overlap_silence_rms")
|
|
if isinstance(overlap_silence_rms, int | float):
|
|
self._overlap_silence_rms = max(0.0, float(overlap_silence_rms))
|
|
|
|
@classmethod
|
|
def _validate_realtime_session_audio_formats(cls, session_payload: dict[str, object]) -> str | None:
|
|
audio_config = session_payload.get("audio")
|
|
input_format: object = session_payload.get("input_audio_format")
|
|
if input_format is None and isinstance(audio_config, dict):
|
|
audio_input = audio_config.get("input")
|
|
if isinstance(audio_input, dict):
|
|
input_format = audio_input.get("format")
|
|
parsed_input, _ = cls._parse_realtime_audio_format(input_format)
|
|
if input_format is not None and not (
|
|
isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
|
):
|
|
return f"Unsupported input_audio_format: {input_format}"
|
|
|
|
output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format")
|
|
if output_format is None and isinstance(audio_config, dict):
|
|
audio_output = audio_config.get("output")
|
|
if isinstance(audio_output, dict):
|
|
output_format = audio_output.get("format")
|
|
parsed_output, _ = cls._parse_realtime_audio_format(output_format)
|
|
if output_format is not None and not (
|
|
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
|
|
):
|
|
return f"Unsupported output_audio_format: {output_format}"
|
|
return None
|
|
|
|
@classmethod
|
|
def _validate_realtime_response_audio_formats(cls, response_payload: dict[str, object]) -> str | None:
|
|
output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format")
|
|
audio_config = response_payload.get("audio")
|
|
if output_format is None and isinstance(audio_config, dict):
|
|
audio_output = audio_config.get("output")
|
|
if isinstance(audio_output, dict):
|
|
output_format = audio_output.get("format")
|
|
parsed_output, _ = cls._parse_realtime_audio_format(output_format)
|
|
if output_format is not None and not (
|
|
isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS
|
|
):
|
|
return f"Unsupported output_audio_format: {output_format}"
|
|
return None
|
|
|
|
@classmethod
|
|
def _validate_conversation_item_audio_formats(cls, item: object) -> str | None:
|
|
if not isinstance(item, dict):
|
|
return None
|
|
content = item.get("content")
|
|
if not isinstance(content, list):
|
|
return None
|
|
for part in content:
|
|
if not isinstance(part, dict):
|
|
continue
|
|
if part.get("type") not in {"input_audio", "audio"}:
|
|
continue
|
|
raw_format = part.get("format")
|
|
parsed_format, _ = cls._parse_realtime_audio_format(raw_format)
|
|
if raw_format is not None and not (
|
|
isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
|
):
|
|
return f"Unsupported input_audio format in conversation.item.create: {raw_format}"
|
|
return None
|
|
|
|
@staticmethod
|
|
def _json_safe_realtime_payload(payload: dict[str, object]) -> dict[str, object]:
|
|
clean: dict[str, object] = {}
|
|
for key, value in payload.items():
|
|
if key == "extra_body":
|
|
continue
|
|
if isinstance(value, str | int | float | bool) or value is None:
|
|
clean[key] = value
|
|
elif isinstance(value, dict):
|
|
clean[key] = RealtimeInputTranslator._json_safe_realtime_payload(value)
|
|
elif isinstance(value, list):
|
|
clean[key] = [
|
|
(RealtimeInputTranslator._json_safe_realtime_payload(item) if isinstance(item, dict) else item)
|
|
for item in value
|
|
if isinstance(item, str | int | float | bool | dict) or item is None
|
|
]
|
|
return clean
|
|
|
|
@staticmethod
|
|
def _parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]:
|
|
def normalize_format(fmt: str) -> str:
|
|
normalized = fmt.lower()
|
|
if normalized in {"audio/pcm", "pcm"}:
|
|
return "pcm16"
|
|
if normalized in {"audio/wav", "wav"}:
|
|
return "wav"
|
|
if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}:
|
|
return "pcm16"
|
|
if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}:
|
|
return "pcm_f32le"
|
|
if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}:
|
|
return "g711_ulaw"
|
|
if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}:
|
|
return "g711_alaw"
|
|
return fmt
|
|
|
|
if isinstance(raw_format, str):
|
|
return normalize_format(raw_format), None
|
|
if not isinstance(raw_format, dict):
|
|
return raw_format, None
|
|
rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate")
|
|
sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None
|
|
fmt = raw_format.get("type") or raw_format.get("format")
|
|
if not isinstance(fmt, str):
|
|
return raw_format, sample_rate_hz
|
|
return normalize_format(fmt), sample_rate_hz
|
|
|
|
@staticmethod
|
|
def _duplex_response_format(realtime_format: str) -> str:
|
|
normalized = realtime_format.lower()
|
|
if normalized in {"pcm16", "pcm_s16le", "s16le"}:
|
|
return "pcm"
|
|
if normalized in {"g711_ulaw", "g711_alaw"}:
|
|
return "pcm"
|
|
if normalized in {"wav", "pcm"}:
|
|
return normalized
|
|
return "wav"
|
|
|
|
@staticmethod
|
|
def _realtime_output_format(duplex_format: object) -> str:
|
|
if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}:
|
|
return duplex_format.lower()
|
|
if isinstance(duplex_format, str) and duplex_format.lower() == "pcm":
|
|
return "pcm16"
|
|
return str(duplex_format or "wav")
|
|
|
|
@staticmethod
|
|
def _is_supported_realtime_input_format(fmt: object) -> bool:
|
|
return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS
|
|
|
|
@staticmethod
|
|
def _input_explicitly_non_speech(event: dict[str, object]) -> bool:
|
|
for key in ("is_speech", "speech"):
|
|
value = event.get(key)
|
|
if isinstance(value, bool):
|
|
return not value
|
|
vad = event.get("vad")
|
|
if isinstance(vad, dict):
|
|
value = vad.get("is_speech")
|
|
if isinstance(value, bool):
|
|
return not value
|
|
probability = vad.get("speech_probability", vad.get("probability"))
|
|
if isinstance(probability, int | float):
|
|
return float(probability) < 0.5
|
|
probability = event.get("speech_probability")
|
|
return isinstance(probability, int | float) and float(probability) < 0.5
|
|
|
|
def _input_looks_like_speech(self, event: dict[str, object], *, audio: object, fmt: object) -> bool:
|
|
if RealtimeInputTranslator._input_explicitly_non_speech(event):
|
|
return False
|
|
for key in ("is_speech", "speech"):
|
|
value = event.get(key)
|
|
if isinstance(value, bool):
|
|
return value
|
|
vad = event.get("vad")
|
|
if isinstance(vad, dict):
|
|
probability = vad.get("speech_probability", vad.get("probability"))
|
|
if isinstance(probability, int | float):
|
|
return float(probability) >= 0.5
|
|
probability = event.get("speech_probability")
|
|
if isinstance(probability, int | float):
|
|
return float(probability) >= 0.5
|
|
if fmt != "pcm_f32le" or not isinstance(audio, str):
|
|
return True
|
|
try:
|
|
raw = base64.b64decode(audio, validate=True)
|
|
except (binascii.Error, ValueError):
|
|
return True
|
|
if len(raw) < 4 or len(raw) % 4 != 0:
|
|
return True
|
|
samples = np.frombuffer(raw, dtype=np.float32)
|
|
if samples.size == 0:
|
|
return False
|
|
rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32)))))
|
|
threshold = event.get("overlap_silence_rms")
|
|
if not isinstance(threshold, int | float):
|
|
vad = event.get("vad")
|
|
if isinstance(vad, dict):
|
|
threshold = vad.get("silence_rms")
|
|
silence_rms = float(threshold) if isinstance(threshold, int | float) else self._overlap_silence_rms
|
|
return rms >= max(0.0, silence_rms)
|
|
|
|
@staticmethod
|
|
def _copy_realtime_input_hints(source: dict[str, object], target: dict[str, object]) -> None:
|
|
for key in (
|
|
"duration_ms",
|
|
"audio_duration_ms",
|
|
"audio_start_ms",
|
|
"audio_end_ms",
|
|
"is_speech",
|
|
"speech",
|
|
"speech_probability",
|
|
"vad",
|
|
"overlap_action",
|
|
"overlap",
|
|
"force_barge_in",
|
|
"force_listen",
|
|
"text",
|
|
"transcript",
|
|
):
|
|
if key in source:
|
|
target[key] = source[key]
|
|
|
|
@classmethod
|
|
def _realtime_overlap_fields(cls, session_payload: dict[str, object]) -> dict[str, object]:
|
|
fields: dict[str, object] = {}
|
|
if isinstance(session_payload.get("overlap_policy"), str):
|
|
fields["overlap_policy"] = session_payload["overlap_policy"]
|
|
for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"):
|
|
value = session_payload.get(key)
|
|
if isinstance(value, int | float):
|
|
fields[key] = value
|
|
|
|
if isinstance(session_payload.get("playback_commit_policy"), str):
|
|
fields["playback_commit_policy"] = session_payload["playback_commit_policy"]
|
|
return fields
|
|
|
|
@staticmethod
|
|
def _validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None:
|
|
"""Validate omni-duplex camera frames on input_audio_buffer.append.
|
|
|
|
Wire contract matches the official MiniCPM-o duplex loop: one base
|
|
base64 JPEG per ~1 s audio chunk, optionally followed by that unit's
|
|
stacked composite tiling the sub-frames captured inside it (at most 2
|
|
images either way). HD slicing (max_slice_nums > 1) is not implemented
|
|
by the duplex adapter yet and is rejected explicitly rather than
|
|
silently ignored; note official suggests it for composites
|
|
(``max_slice_nums=[2, 1]``), so stacked detail is capped at
|
|
scale_resolution until that lands.
|
|
"""
|
|
if max_slice_nums not in (None, 1):
|
|
return "max_slice_nums > 1 (HD slicing) is not implemented by the duplex Realtime adapter"
|
|
if not isinstance(video_frames, list):
|
|
return "video_frames must be a list of base64-encoded images"
|
|
frames = [frame for frame in video_frames if frame is not None]
|
|
if len(frames) > 2:
|
|
return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk"
|
|
for frame in frames:
|
|
if not isinstance(frame, str) or not frame:
|
|
return "video_frames entries must be non-empty base64 strings"
|
|
if len(frame) > 4_000_000:
|
|
return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality"
|
|
try:
|
|
header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4))
|
|
except (binascii.Error, ValueError):
|
|
return "video_frames entries must be valid base64"
|
|
if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")):
|
|
return "video_frames entries must be JPEG or PNG images"
|
|
return None
|
|
|
|
@staticmethod
|
|
def _configured_realtime_turn_detection(session_payload: dict[str, object]) -> tuple[str | None, object]:
|
|
field = "turn_detection" if "turn_detection" in session_payload else None
|
|
selected = session_payload.get("turn_detection")
|
|
audio_config = session_payload.get("audio")
|
|
if isinstance(audio_config, dict):
|
|
audio_input = audio_config.get("input")
|
|
if isinstance(audio_input, dict) and "turn_detection" in audio_input:
|
|
field, selected = "audio.input.turn_detection", audio_input["turn_detection"]
|
|
return field, selected
|
|
|
|
@classmethod
|
|
def _validate_realtime_turn_detection(cls, session_payload: dict[str, object]) -> str | None:
|
|
field, turn_detection = cls._configured_realtime_turn_detection(session_payload)
|
|
if field is None:
|
|
if session_payload.get("overlap_policy") == "barge_in_on_speech":
|
|
return (
|
|
"overlap_policy='barge_in_on_speech' requires "
|
|
"turn_detection.type='server_vad' on the Realtime endpoint"
|
|
)
|
|
return None
|
|
if turn_detection is not None and not isinstance(turn_detection, dict):
|
|
return f"{field} must be null or an object with type='server_vad'"
|
|
if isinstance(turn_detection, dict):
|
|
if turn_detection.get("type") != "server_vad":
|
|
return f"{field}.type must be 'server_vad'"
|
|
threshold = turn_detection.get("threshold", 0.5)
|
|
if not cls._valid_vad_number(threshold, minimum=SILERO_VAD_MIN_THRESHOLD, strict=True, maximum=1):
|
|
return f"{field}.threshold must be greater than {SILERO_VAD_MIN_THRESHOLD} and at most 1"
|
|
for name in ("prefix_padding_ms", "silence_duration_ms", "min_speech_duration_ms"):
|
|
value = turn_detection.get(name)
|
|
if value is not None and not cls._valid_vad_number(value, minimum=0):
|
|
return f"{field}.{name} must be a non-negative number"
|
|
if turn_detection.get("interrupt_response", True) is not True:
|
|
return (
|
|
f"{field}.interrupt_response=false is unsupported; "
|
|
"use turn_detection=null for model-owned listen/speak"
|
|
)
|
|
desired_policy = "barge_in_on_speech" if turn_detection is not None else "listen_only"
|
|
overlap_policy = session_payload.get("overlap_policy")
|
|
if isinstance(overlap_policy, str) and overlap_policy != desired_policy:
|
|
return f"overlap_policy={overlap_policy!r} conflicts with turn_detection; expected {desired_policy!r}"
|
|
return None
|
|
|
|
@staticmethod
|
|
def _valid_vad_number(value: object, *, minimum: float, maximum: float | None = None, strict: bool = False) -> bool:
|
|
return (
|
|
isinstance(value, int | float)
|
|
and not isinstance(value, bool)
|
|
and np.isfinite(float(value))
|
|
and (float(value) > minimum if strict else float(value) >= minimum)
|
|
and (maximum is None or float(value) <= maximum)
|
|
)
|
|
|
|
def _prepare_realtime_turn_detection_update(self, session_payload: dict[str, object]) -> None:
|
|
field, configured = self._configured_realtime_turn_detection(session_payload)
|
|
if field is None:
|
|
return
|
|
if self._pending_turn_detection_update is not None:
|
|
raise RuntimeError("A Realtime turn-detection update is already pending")
|
|
|
|
turn_detection: dict[str, object] | None = None
|
|
detector: SileroStreamingVAD | None = None
|
|
if configured is None:
|
|
session_payload["overlap_policy"] = "listen_only"
|
|
else:
|
|
assert isinstance(configured, dict)
|
|
turn_detection = {
|
|
"type": "server_vad",
|
|
"interrupt_response": True,
|
|
"threshold": 0.5,
|
|
"prefix_padding_ms": 300,
|
|
"silence_duration_ms": 500,
|
|
"min_speech_duration_ms": 96,
|
|
**configured,
|
|
}
|
|
detector = SileroStreamingVAD(
|
|
SileroVADConfig(
|
|
threshold=float(turn_detection["threshold"]),
|
|
prefix_padding_ms=int(turn_detection["prefix_padding_ms"]),
|
|
silence_duration_ms=int(turn_detection["silence_duration_ms"]),
|
|
min_speech_duration_ms=max(32, int(turn_detection["min_speech_duration_ms"])),
|
|
)
|
|
)
|
|
session_payload["turn_detection"] = dict(turn_detection)
|
|
session_payload["overlap_policy"] = "barge_in_on_speech"
|
|
|
|
self._pending_turn_detection_update = (turn_detection, detector, asyncio.Event())
|
|
|
|
async def wait_for_realtime_turn_detection_update(self) -> None:
|
|
candidate = self._pending_turn_detection_update
|
|
if candidate is not None:
|
|
await candidate[2].wait()
|
|
|
|
def commit_realtime_turn_detection_update(self) -> None:
|
|
self._resolve_realtime_turn_detection_update(commit=True)
|
|
|
|
def reject_realtime_turn_detection_update(self) -> None:
|
|
self._resolve_realtime_turn_detection_update(commit=False)
|
|
|
|
def _resolve_realtime_turn_detection_update(self, *, commit: bool) -> None:
|
|
candidate = self._pending_turn_detection_update
|
|
if candidate is None:
|
|
return
|
|
turn_detection, detector, resolved = candidate
|
|
discarded = self._server_vad if commit else detector
|
|
if commit:
|
|
self._turn_detection, self._server_vad = turn_detection, detector
|
|
self._pending_turn_detection_update = None
|
|
resolved.set()
|
|
reset = getattr(discarded, "reset", None)
|
|
if callable(reset):
|
|
reset()
|
|
|
|
def _reset_server_vad(self) -> None:
|
|
reset = getattr(self._server_vad, "reset", None)
|
|
if callable(reset):
|
|
reset()
|
|
|
|
@staticmethod
|
|
def _input_audio_transcription_config(session_payload: dict[str, object]) -> dict[str, object] | None:
|
|
transcription = session_payload.get("input_audio_transcription")
|
|
if isinstance(transcription, dict):
|
|
return transcription
|
|
audio_config = session_payload.get("audio")
|
|
if not isinstance(audio_config, dict):
|
|
return None
|
|
audio_input = audio_config.get("input")
|
|
if not isinstance(audio_input, dict):
|
|
return None
|
|
transcription = audio_input.get("transcription")
|
|
return transcription if isinstance(transcription, dict) else None
|
|
|
|
@staticmethod
|
|
def realtime_max_output_tokens(value: object) -> int | None:
|
|
"""Normalize Realtime max output tokens.
|
|
|
|
OpenAI Realtime clients commonly send ``"inf"`` for unbounded output.
|
|
The duplex core represents that as ``None`` because model-specific
|
|
defaults remain in force.
|
|
"""
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}:
|
|
return None
|
|
if isinstance(value, int) and value > 0:
|
|
return int(value)
|
|
return None
|
|
|
|
async def _conversation_item_to_duplex(self, event: dict[str, object]) -> dict[str, object] | None:
|
|
item = event.get("item")
|
|
if not isinstance(item, dict):
|
|
return None
|
|
item = self._normalize_conversation_item(item)
|
|
previous_item_id = event.get("previous_item_id")
|
|
if isinstance(previous_item_id, str):
|
|
item["_previous_item_id"] = previous_item_id
|
|
item_id = str(item["id"])
|
|
item_type = item.get("type")
|
|
role = item.get("role")
|
|
if item_type == "function_call_output":
|
|
call_id = item.get("call_id")
|
|
output = item.get("output")
|
|
matching_call = any(
|
|
known.get("type") == "function_call" and known.get("call_id") == call_id
|
|
for known in self._conversation_items.values()
|
|
)
|
|
duplicate_output = any(
|
|
known.get("type") == "function_call_output" and known.get("call_id") == call_id
|
|
for known in self._conversation_items.values()
|
|
)
|
|
if not isinstance(call_id, str) or not call_id or not matching_call:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"invalid_function_call_output",
|
|
"function_call_output requires the call_id of a completed function call",
|
|
event_id=event.get("event_id"),
|
|
param="item.call_id",
|
|
)
|
|
)
|
|
return None
|
|
if not isinstance(output, str) or duplicate_output:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"invalid_function_call_output",
|
|
(
|
|
"function_call_output requires a string output"
|
|
if not isinstance(output, str)
|
|
else f"function_call_output already exists for call_id {call_id}"
|
|
),
|
|
event_id=event.get("event_id"),
|
|
param="item.output",
|
|
)
|
|
)
|
|
return None
|
|
# Do not retain the result until SessionRunner has delivered its
|
|
# runtime update. On success, the resulting
|
|
# ``conversation.item.created`` event records it in the shared
|
|
# input/output protocol state; on failure the client may retry the
|
|
# same call_id instead of being rejected by provisional history.
|
|
if item_type != "message" or role in {"assistant", "system"}:
|
|
return {
|
|
"type": "turn.signal",
|
|
"event": "conversation.item.create",
|
|
"payload": {"item": item},
|
|
}
|
|
self._conversation_items[item_id] = item
|
|
content = item.get("content")
|
|
if not isinstance(content, list):
|
|
return None
|
|
text_chunks: list[str] = []
|
|
audio_events: list[dict[str, object]] = []
|
|
for part in content:
|
|
if not isinstance(part, dict):
|
|
continue
|
|
if part.get("type") in {"input_text", "text"} and isinstance(part.get("text"), str):
|
|
text_chunks.append(str(part["text"]))
|
|
if part.get("type") in {"input_audio", "audio"}:
|
|
audio = part.get("audio") or part.get("data")
|
|
fmt, format_rate = self._parse_realtime_audio_format(part.get("format") or self._input_audio_format)
|
|
sample_rate_hz = (
|
|
part.get("sample_rate_hz") or part.get("sample_rate") or format_rate or self._input_sample_rate_hz
|
|
)
|
|
if not self._is_supported_realtime_input_format(fmt):
|
|
continue
|
|
try:
|
|
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
|
|
audio,
|
|
fmt,
|
|
sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None,
|
|
)
|
|
except ValueError as exc:
|
|
await self._send_realtime_payload(
|
|
self._realtime_error_payload(
|
|
"bad_event",
|
|
str(exc),
|
|
event_id=event.get("event_id"),
|
|
param="sample_rate_hz",
|
|
)
|
|
)
|
|
return None
|
|
if not isinstance(audio, str) or not audio:
|
|
continue
|
|
speech_hints = dict(event)
|
|
speech_hints.update(part)
|
|
if not self._input_looks_like_speech(speech_hints, audio=audio, fmt=fmt):
|
|
continue
|
|
self._input_speech_started = True
|
|
payload = {
|
|
"type": "input_audio_buffer.append",
|
|
"audio": audio,
|
|
"format": fmt,
|
|
"sample_rate_hz": sample_rate_hz,
|
|
}
|
|
self._copy_realtime_input_hints(part, payload)
|
|
self._copy_realtime_input_hints(event, payload)
|
|
audio_events.append(payload)
|
|
if audio_events:
|
|
self._input_audio_buffer_has_audio = True
|
|
transcript = self._input_transcript_from_item(item)
|
|
for extra_event in audio_events[1:]:
|
|
self._pending_outbound.put_nowait(extra_event)
|
|
commit_payload: dict[str, object] = {
|
|
"type": "input_audio_buffer.commit",
|
|
"final": True,
|
|
"realtime_item_id": item_id,
|
|
"response_create": False,
|
|
}
|
|
if transcript:
|
|
commit_payload["transcript"] = transcript
|
|
self._pending_outbound.put_nowait(commit_payload)
|
|
return audio_events[0]
|
|
if not text_chunks:
|
|
return None
|
|
text_item = dict(item)
|
|
text_item["status"] = "completed"
|
|
return {
|
|
"type": "turn.signal",
|
|
"event": "conversation.item.create",
|
|
"payload": {"item": text_item},
|
|
}
|
|
|
|
def _remember_input_transcript_hint(self, event: dict[str, object]) -> None:
|
|
transcript = event.get("transcript")
|
|
if not isinstance(transcript, str):
|
|
transcript = event.get("text") if isinstance(event.get("text"), str) else None
|
|
if not isinstance(transcript, str):
|
|
hints = event.get("hints")
|
|
if isinstance(hints, dict):
|
|
transcript = hints.get("transcript")
|
|
if not isinstance(transcript, str):
|
|
transcript = hints.get("text") if isinstance(hints.get("text"), str) else None
|
|
if isinstance(transcript, str) and transcript:
|
|
if (
|
|
self._input_audio_buffer_transcript_parts
|
|
and self._input_audio_buffer_transcript_parts[-1] == transcript
|
|
):
|
|
return
|
|
self._input_audio_buffer_transcript_parts.append(transcript)
|
|
|
|
def _consume_input_transcript_hint(self) -> str:
|
|
transcript = "".join(self._input_audio_buffer_transcript_parts).strip()
|
|
self._input_audio_buffer_transcript_parts.clear()
|
|
return transcript
|
|
|
|
@staticmethod
|
|
def _input_transcript_from_item(item: dict[str, object]) -> str:
|
|
content = item.get("content")
|
|
if not isinstance(content, list):
|
|
return ""
|
|
parts: list[str] = []
|
|
for part in content:
|
|
if not isinstance(part, dict):
|
|
continue
|
|
for key in ("transcript", "text"):
|
|
value = part.get(key)
|
|
if isinstance(value, str) and value:
|
|
if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}:
|
|
parts.append(value)
|
|
break
|
|
return "".join(parts).strip()
|
|
|
|
@staticmethod
|
|
def _input_audio_transcription_completed_event(
|
|
item_id: str,
|
|
item: dict[str, object],
|
|
) -> dict[str, object] | None:
|
|
content = item.get("content")
|
|
if not isinstance(content, list):
|
|
return None
|
|
transcript_parts: list[str] = []
|
|
for index, part in enumerate(content):
|
|
if not isinstance(part, dict):
|
|
continue
|
|
if part.get("type") != "input_audio":
|
|
continue
|
|
transcript = part.get("transcript")
|
|
if isinstance(transcript, str) and transcript:
|
|
transcript_parts.append(transcript)
|
|
return {
|
|
"type": "conversation.item.input_audio_transcription.completed",
|
|
"item_id": item_id,
|
|
"content_index": index,
|
|
"transcript": "".join(transcript_parts),
|
|
}
|
|
return None
|
|
|
|
def _pop_pending_commit_item_id(self) -> str:
|
|
try:
|
|
return self._pending_commit_item_ids.get_nowait()
|
|
except asyncio.QueueEmpty:
|
|
return f"item_{uuid4().hex}"
|
|
|
|
async def _emit_input_speech_started(self, event: dict[str, object]) -> None:
|
|
if self._input_speech_started:
|
|
return
|
|
self._input_speech_started = True
|
|
if self._active_input_item_id is None:
|
|
self._active_input_item_id = f"item_{uuid4().hex}"
|
|
audio_start_ms = event.get("audio_start_ms", 0)
|
|
await self._send_realtime_payload(
|
|
{
|
|
"type": "input_audio_buffer.speech_started",
|
|
"audio_start_ms": int(audio_start_ms) if isinstance(audio_start_ms, int | float) else 0,
|
|
"item_id": self._active_input_item_id,
|
|
}
|
|
)
|
|
|
|
async def _emit_input_speech_stopped(self, event: dict[str, object], *, item_id: str) -> None:
|
|
if not self._input_speech_started:
|
|
return
|
|
self._input_speech_started = False
|
|
audio_end_ms = event.get("audio_end_ms", event.get("audio_ms", 0))
|
|
await self._send_realtime_payload(
|
|
{
|
|
"type": "input_audio_buffer.speech_stopped",
|
|
"audio_end_ms": int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0,
|
|
"item_id": item_id,
|
|
}
|
|
)
|