# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import asyncio import base64 import binascii from collections.abc import Callable from typing import Any from uuid import uuid4 import numpy as np from vllm_omni.entrypoints.duplex.audio import ( convert_input_audio_with_rate, ) from vllm_omni.entrypoints.duplex.protocol import normalize_native_duplex_key from vllm_omni.entrypoints.duplex.realtime_state import ( REALTIME_INPUT_AUDIO_FORMATS, REALTIME_OUTPUT_AUDIO_FORMATS, RealtimeStateOwner, ) from vllm_omni.entrypoints.duplex.vad import ( SILERO_VAD_MIN_THRESHOLD, ServerVADUnavailableError, SileroStreamingVAD, SileroVADConfig, StreamingVADResult, ) class RealtimeInputTranslator(RealtimeStateOwner): """Translate and validate client Realtime events for the duplex core.""" _opened: bool _initial_session_update: bool _input_speech_started: bool _input_audio_buffer_has_audio: bool _input_audio_buffer_had_non_speech: bool _input_audio_buffer_transcript_parts: list[str] _active_input_item_id: str | None _last_conversation_item_id: str | None _active_response_id: str | None _last_response_id: str | None _input_audio_format: str _output_audio_format: str _input_sample_rate_hz: int _conversation_items: dict[str, dict[str, object]] _item_truncation_cursors: dict[str, tuple[int, int]] _pending_outbound: asyncio.Queue[dict[str, object]] _pending_commit_item_ids: asyncio.Queue[str] async def _send_realtime_payload(self, payload: dict[str, object]) -> None: raise NotImplementedError def _realtime_error_payload( self, code: str, message: str, *, event_id: object | None = None, param: object | None = None, ) -> dict[str, object]: raise NotImplementedError _response_is_done: Callable[[object], bool] async def discard_pending_input_audio( self, *, audio_end_ms: int | None = None, ) -> None: """Drop Realtime input-buffer state that was consumed as overlap. The serving overlap policy may classify a short acknowledgement such as "continue" as a side-channel signal instead of user turn input. The Realtime protocol has already observed the append and may have emitted ``speech_started``; reset that transient buffer so a later commit does not accidentally attach the acknowledgement transcript/audio to the next real user turn. """ if self._input_speech_started and self._active_input_item_id is not None: await self._send_realtime_payload( { "type": "input_audio_buffer.speech_stopped", "audio_end_ms": max(0, int(audio_end_ms or 0)), "item_id": self._active_input_item_id, } ) self._input_speech_started = False self._active_input_item_id = None self._input_audio_buffer_has_audio = False self._input_audio_buffer_had_non_speech = False self._input_audio_buffer_transcript_parts.clear() async def _send_realtime_input_ack(self, event: dict[str, object]) -> None: if event.get("type") != "conversation.item.create": return item = event.get("item") if not isinstance(item, dict): return item = self._normalize_conversation_item(item) previous_item_id = event.get("previous_item_id") if isinstance(previous_item_id, str): item["_previous_item_id"] = previous_item_id if item.get("role") != "user": return self._conversation_items[str(item["id"])] = item for payload in self._conversation_item_added_events(item): await self._send_realtime_payload(payload) async def _to_duplex_event(self, event: dict[str, object]) -> dict[str, object] | None: event_type = event.get("type") if event_type == "session.update": session = event.get("session") session_payload: dict[str, object] = session if isinstance(session, dict) else event format_error = self._validate_realtime_session_audio_formats(session_payload) if format_error is not None: await self._send_realtime_payload( self._realtime_error_payload( "unsupported_audio_format", format_error, event_id=event.get("event_id"), ) ) return None turn_detection_error = self._validate_realtime_turn_detection(session_payload) if turn_detection_error is not None: await self._send_realtime_payload( self._realtime_error_payload( "unsupported_turn_detection", turn_detection_error, event_id=event.get("event_id"), param="turn_detection", ) ) return None self._prepare_realtime_turn_detection_update(session_payload) self._apply_realtime_session_defaults(session_payload) session_payload.update(self._realtime_overlap_fields(session_payload)) if not self._opened: self._opened = True self._initial_session_update = True return self._session_create_from_realtime(session_payload) return { "type": "turn.signal", "event": "session.update", "payload": session_payload, "realtime_event_id": event.get("event_id"), } if event_type == "conversation.item.create": item = event.get("item") format_error = self._validate_conversation_item_audio_formats(item) if format_error is not None: await self._send_realtime_payload( self._realtime_error_payload( "unsupported_audio_format", format_error, event_id=event.get("event_id"), ) ) return None return await self._conversation_item_to_duplex(event) if event_type == "conversation.item.delete": item_id = event.get("item_id") if not isinstance(item_id, str) or not item_id: await self._send_realtime_payload( self._realtime_error_payload( "missing_item_id", "conversation.item.delete requires item_id", event_id=event.get("event_id"), ) ) return None if item_id not in self._conversation_items: await self._send_realtime_payload( self._realtime_error_payload( "item_not_found", f"Conversation item not found: {item_id}", event_id=event.get("event_id"), ) ) return None self._remove_conversation_item(item_id) await self._pending_outbound.put( { "type": "turn.signal", "event": "conversation.item.delete", "payload": {"item_id": item_id}, } ) return None if event_type == "conversation.item.retrieve": item_id = event.get("item_id") if not isinstance(item_id, str) or not item_id: await self._send_realtime_payload( self._realtime_error_payload( "missing_item_id", "conversation.item.retrieve requires item_id", event_id=event.get("event_id"), ) ) return None item = self._conversation_items.get(item_id) if item is None: await self._send_realtime_payload( self._realtime_error_payload( "item_not_found", f"Conversation item not found: {item_id}", event_id=event.get("event_id"), ) ) return None await self._send_realtime_payload({"type": "conversation.item.retrieved", "item": item}) return None if event_type == "conversation.item.truncate": item_id = event.get("item_id") audio_end_ms = event.get("audio_end_ms") content_index = event.get("content_index", 0) if not isinstance(item_id, str) or not item_id: await self._send_realtime_payload( self._realtime_error_payload( "missing_item_id", "conversation.item.truncate requires item_id", event_id=event.get("event_id"), ) ) return None if not isinstance(audio_end_ms, int | float): await self._send_realtime_payload( self._realtime_error_payload( "bad_event", "conversation.item.truncate requires numeric audio_end_ms", event_id=event.get("event_id"), param="audio_end_ms", ) ) return None item = self._conversation_items.get(item_id) if not isinstance(item, dict): await self._send_realtime_payload( self._realtime_error_payload( "item_not_found", f"Conversation item not found: {item_id}", event_id=event.get("event_id"), ) ) return None truncate_error = self._validate_realtime_item_truncate( item, content_index=int(content_index) if isinstance(content_index, int | float) else 0, audio_end_ms=int(audio_end_ms), ) if truncate_error is not None: await self._send_realtime_payload( self._realtime_error_payload( "bad_event", truncate_error, event_id=event.get("event_id"), ) ) return None self._item_truncation_cursors[item_id] = ( int(content_index) if isinstance(content_index, int | float) else 0, int(audio_end_ms), ) self._truncate_realtime_item_content( item, content_index=int(content_index) if isinstance(content_index, int | float) else 0, audio_end_ms=int(audio_end_ms), ) await self._pending_outbound.put( { "type": "turn.signal", "event": "conversation.item.truncate", "payload": { "item_id": item_id, "content_index": content_index, "audio_end_ms": audio_end_ms, }, } ) await self._pending_outbound.put( { "type": "playback.ack", "item_id": item_id, "committed_ms": int(audio_end_ms), "played_ms": int(audio_end_ms), "truncate": True, } ) return None if event_type == "input_audio_buffer.append": audio = event.get("audio") or event.get("delta") fmt, format_rate = self._parse_realtime_audio_format( event.get("format") or event.get("input_audio_format") or self._input_audio_format ) sample_rate_hz = ( event.get("sample_rate_hz") or event.get("sample_rate") or format_rate or self._input_sample_rate_hz ) if not self._is_supported_realtime_input_format(fmt): await self._send_realtime_payload( self._realtime_error_payload( "unsupported_audio_format", f"Unsupported input_audio_format: {fmt}", event_id=event.get("event_id"), ) ) return None try: audio, fmt, sample_rate_hz = convert_input_audio_with_rate( audio, fmt, sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None, ) except ValueError as exc: await self._send_realtime_payload( self._realtime_error_payload( "bad_event", str(exc), event_id=event.get("event_id"), param="sample_rate_hz", ) ) return None detector = self._server_vad vad_result: StreamingVADResult | None = None if isinstance(detector, SileroStreamingVAD): try: vad_result = detector.process_base64(audio, fmt=fmt, sample_rate_hz=sample_rate_hz) except (ServerVADUnavailableError, ValueError) as exc: unavailable = isinstance(exc, ServerVADUnavailableError) await self._send_realtime_payload( self._realtime_error_payload( "server_vad_unavailable" if unavailable else "bad_audio", str(exc), event_id=event.get("event_id"), param="turn_detection" if unavailable else "audio", ) ) return None looks_like_speech = ( vad_result.is_speech if vad_result is not None else self._input_looks_like_speech(event, audio=audio, fmt=fmt) ) self._input_audio_buffer_has_audio = self._input_audio_buffer_has_audio or ( looks_like_speech and isinstance(audio, str) and bool(audio) ) self._input_audio_buffer_had_non_speech = self._input_audio_buffer_had_non_speech or ( not looks_like_speech and isinstance(audio, str) and bool(audio) ) stopped_event: dict[str, object] | None = None if vad_result is not None and vad_result.speech_stopped: stopped_event = dict(event) if vad_result.speech_end_ms is not None: stopped_event["audio_end_ms"] = vad_result.speech_end_ms if stopped_event is not None and vad_result is not None and vad_result.speech_active: await self._emit_input_speech_stopped( stopped_event, item_id=self._active_input_item_id or f"item_{uuid4().hex}", ) if vad_result is not None and vad_result.speech_started: started_event = dict(event) if vad_result.speech_start_ms is not None: started_event["audio_start_ms"] = vad_result.speech_start_ms await self._emit_input_speech_started(started_event) elif vad_result is None and looks_like_speech: await self._emit_input_speech_started(event) if stopped_event is not None and vad_result is not None and not vad_result.speech_active: await self._emit_input_speech_stopped( stopped_event, item_id=self._active_input_item_id or f"item_{uuid4().hex}", ) if looks_like_speech: self._remember_input_transcript_hint(event) payload = { "type": "input_audio_buffer.append", "audio": audio, "format": fmt, "sample_rate_hz": sample_rate_hz, } video_frames = event.get("video_frames") if video_frames is not None: frames_error = self._validate_realtime_video_frames(video_frames, event.get("max_slice_nums")) if frames_error is not None: await self._send_realtime_payload( self._realtime_error_payload( "invalid_video_frames", frames_error, event_id=event.get("event_id"), param="video_frames", ) ) return None if isinstance(video_frames, list): payload["video_frames"] = [frame for frame in video_frames if isinstance(frame, str) and frame] self._copy_realtime_input_hints(event, payload) payload["is_speech"] = looks_like_speech if vad_result is not None: payload["vad"] = { "backend": "silero", "is_speech": vad_result.is_speech, "speech_active": vad_result.speech_active, "speech_started": vad_result.speech_started, "speech_stopped": vad_result.speech_stopped, "speech_probability": vad_result.speech_probability, } if vad_result.speech_active: payload["force_listen"] = True return payload if event_type == "input_audio_buffer.commit": if not self._input_audio_buffer_has_audio: if self._input_audio_buffer_had_non_speech: self._input_audio_buffer_had_non_speech = False self._active_input_item_id = None self._reset_server_vad() return { "type": "input_audio_buffer.commit", "final": event.get("final", True), "response_create": False, "is_speech": False, } await self._send_realtime_payload( self._realtime_error_payload( "input_audio_buffer_empty", "input_audio_buffer.commit requires a non-empty input audio buffer", event_id=event.get("event_id"), ) ) return None item_id = self._active_input_item_id or f"item_{uuid4().hex}" await self._pending_commit_item_ids.put(item_id) await self._emit_input_speech_stopped(event, item_id=item_id) transcript = self._consume_input_transcript_hint() self._active_input_item_id = None self._input_audio_buffer_has_audio = False self._input_audio_buffer_had_non_speech = False self._reset_server_vad() payload = { "type": "input_audio_buffer.commit", "final": event.get("final", True), "realtime_item_id": item_id, "response_create": bool(event.get("response_create", False)), } if transcript: payload["transcript"] = transcript return payload if event_type == "input_audio_buffer.clear": self._input_speech_started = False self._active_input_item_id = None self._input_audio_buffer_has_audio = False self._input_audio_buffer_had_non_speech = False self._input_audio_buffer_transcript_parts.clear() self._reset_server_vad() return {"type": "input_audio_buffer.clear", "reason": event_type} if event_type == "output_audio_buffer.clear": payload = {"type": "output_audio_buffer.clear", "reason": event_type} response_id = event.get("response_id") if not isinstance(response_id, str) or not response_id: response_id = self._active_response_id or self._last_response_id if self._response_is_done(response_id): await self._send_realtime_payload( { "type": "output_audio_buffer.cleared", "response_id": response_id, } ) return None if isinstance(response_id, str) and response_id: payload["response_id"] = response_id return payload if event_type == "response.cancel": payload = {"type": "response.cancel", "reason": event_type} response_id = event.get("response_id") if not isinstance(response_id, str) or not response_id: response_id = self._active_response_id or self._last_response_id if self._response_is_done(response_id): await self._send_realtime_payload( self._realtime_error_payload( "response_not_active", f"Response is already complete: {response_id}", event_id=event.get("event_id"), ) ) return None if isinstance(response_id, str) and response_id: payload["response_id"] = response_id return payload if event_type == "response.create": response_payload = event.get("response") if isinstance(response_payload, dict): format_error = self._validate_realtime_response_audio_formats(response_payload) if format_error is not None: await self._send_realtime_payload( self._realtime_error_payload( "unsupported_audio_format", format_error, event_id=event.get("event_id"), ) ) return None return { "type": "response.create", "response": response_payload if isinstance(response_payload, dict) else {}, } if event_type in {"session.close", "close"}: return {"type": "session.close"} return event @staticmethod def _normalize_conversation_item(item: dict[str, object]) -> dict[str, object]: normalized = dict(item) normalized.setdefault("id", f"item_{uuid4().hex}") normalized.setdefault("object", "realtime.item") normalized.setdefault("type", "message") normalized.setdefault("status", "completed") if "role" not in normalized and normalized.get("type") == "message": normalized["role"] = "user" if not isinstance(normalized.get("content"), list): normalized["content"] = [] return normalized @staticmethod def _truncate_realtime_item_content( item: dict[str, object], *, content_index: int, audio_end_ms: int, ) -> None: content = item.get("content") if not isinstance(content, list) or not content: return index = max(0, int(content_index)) if index >= len(content): return part = content[index] if not isinstance(part, dict): return transcript = part.get("transcript") if not isinstance(transcript, str) or not transcript: return marks = part.get("audio_text_marks") if isinstance(marks, list): keep_chars = RealtimeInputTranslator._text_chars_for_audio_ms_from_marks( audio_end_ms, len(transcript), marks, final_ms=part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms"), ) part["transcript"] = transcript[:keep_chars].rstrip() return duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") if isinstance(duration_ms, int | float) and duration_ms > 0: keep_chars = int(len(transcript) * max(0.0, min(1.0, int(audio_end_ms) / float(duration_ms)))) part["transcript"] = transcript[:keep_chars].rstrip() elif audio_end_ms <= 0: part["transcript"] = "" @staticmethod def _validate_realtime_item_truncate( item: dict[str, object], *, content_index: int, audio_end_ms: int, ) -> str | None: if item.get("type") != "message" or item.get("role") != "assistant": return "conversation.item.truncate only supports assistant message items" if audio_end_ms < 0: return "conversation.item.truncate requires non-negative audio_end_ms" content = item.get("content") if not isinstance(content, list) or not content: return None index = max(0, int(content_index)) if index >= len(content): return f"conversation.item.truncate content_index out of range: {content_index}" part = content[index] if not isinstance(part, dict): return "conversation.item.truncate target content part is invalid" if part.get("type") not in {"audio", "output_audio"}: return "conversation.item.truncate target content part must be audio" duration_ms = part.get("audio_duration_ms") or part.get("duration_ms") or part.get("audio_ms") if isinstance(duration_ms, int | float) and int(duration_ms) >= 0 and audio_end_ms > int(duration_ms): return "conversation.item.truncate audio_end_ms exceeds item audio duration" return None @staticmethod def _text_chars_for_audio_ms_from_marks( audio_end_ms: int, text_len: int, marks: list[object], *, final_ms: object | None = None, ) -> int: if text_len <= 0: return 0 clean_marks: list[tuple[int, int]] = [] for mark in marks: if not isinstance(mark, dict): continue raw_text_chars = mark.get("text_chars") raw_audio_end_ms = mark.get("audio_end_ms", mark.get("audio_ms")) if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float): continue clean_marks.append((max(0, int(raw_audio_end_ms)), min(text_len, max(0, int(raw_text_chars))))) if not clean_marks: return 0 if audio_end_ms <= 0 else text_len clean_marks.sort(key=lambda item: item[0]) audio_end_ms = max(0, int(audio_end_ms)) if audio_end_ms <= 0: return 0 previous_ms = 0 previous_chars = 0 for mark_ms, mark_chars in clean_marks: mark_ms = max(previous_ms, mark_ms) mark_chars = max(previous_chars, min(text_len, mark_chars)) if audio_end_ms <= mark_ms: if mark_ms <= previous_ms: return mark_chars ratio = (audio_end_ms - previous_ms) / max(1, mark_ms - previous_ms) return int(previous_chars + (mark_chars - previous_chars) * max(0.0, min(1.0, ratio))) previous_ms = mark_ms previous_chars = mark_chars if isinstance(final_ms, int | float) and int(final_ms) > previous_ms: if audio_end_ms >= int(final_ms): return text_len ratio = (audio_end_ms - previous_ms) / max(1, int(final_ms) - previous_ms) return int(previous_chars + (text_len - previous_chars) * max(0.0, min(1.0, ratio))) return text_len if audio_end_ms >= previous_ms else previous_chars def _conversation_item_added_events(self, item: dict[str, object]) -> list[dict[str, object]]: item_id = item.get("id") explicit_previous_item_id = item.pop("_previous_item_id", None) previous_item_id = ( explicit_previous_item_id if isinstance(explicit_previous_item_id, str) else self._last_conversation_item_id ) if isinstance(item_id, str) and item_id: self._last_conversation_item_id = item_id return [ { "type": "conversation.item.added", "previous_item_id": previous_item_id, "item": item, }, { "type": "conversation.item.created", "previous_item_id": previous_item_id, "item": item, }, ] def _conversation_item_done_event(self, item: dict[str, object]) -> dict[str, object]: item_id = item.get("id") previous_item_id = self._previous_item_id(item_id) if isinstance(item_id, str) else None if isinstance(item_id, str) and item_id: self._conversation_items[item_id] = item self._last_conversation_item_id = item_id return { "type": "conversation.item.done", "previous_item_id": previous_item_id, "item": item, } def _remove_conversation_item(self, item_id: str) -> bool: removed = self._conversation_items.pop(item_id, None) is not None if self._last_conversation_item_id == item_id: remaining_ids = list(self._conversation_items) self._last_conversation_item_id = remaining_ids[-1] if remaining_ids else None self._item_truncation_cursors.pop(item_id, None) return removed def _response_output_item_added_events( self, *, response_id: object, item: dict[str, object], ) -> list[dict[str, object]]: return [ { "type": "response.output_item.added", "response_id": response_id, "output_index": 0, "item": item, }, ] @staticmethod def _response_content_part(*, transcript: str = "") -> dict[str, object]: return { "type": "audio", "transcript": transcript, } @staticmethod def _response_text_content_part(*, text: str = "") -> dict[str, object]: return { "type": "text", "text": text, } @staticmethod def _response_item_content_part( *, transcript: str = "", audio_duration_ms: int | None = None, audio_text_marks: list[dict[str, int]] | None = None, ) -> dict[str, object]: part: dict[str, object] = { "type": "output_audio", "transcript": transcript, } if audio_duration_ms is not None: part["audio_duration_ms"] = int(audio_duration_ms) if audio_text_marks: part["audio_text_marks"] = [dict(mark) for mark in audio_text_marks] return part @staticmethod def _response_item_text_content_part(*, text: str = "") -> dict[str, object]: return { "type": "output_text", "text": text, } def _previous_item_id(self, item_id: str) -> str | None: previous: str | None = None for known_id in self._conversation_items: if known_id == item_id: return previous previous = known_id if self._last_conversation_item_id == item_id: return previous return self._last_conversation_item_id def _input_audio_buffer_committed_event( self, *, item_id: str, event: dict[str, Any], ) -> dict[str, object]: return { "type": "input_audio_buffer.committed", "previous_item_id": self._previous_item_id(item_id), "item_id": item_id, "event": event, } def _session_create_from_realtime(self, session_payload: dict[str, object]) -> dict[str, object]: self._apply_realtime_session_defaults(session_payload) model = session_payload.get("model") audio_config = session_payload.get("audio") audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None extra_body_payload = session_payload.get("extra_body") extra_body = ( normalize_native_duplex_key(dict(extra_body_payload)) if isinstance(extra_body_payload, dict) else {} ) extra_body["realtime_session_payload"] = self._json_safe_realtime_payload(session_payload) if isinstance(session_payload.get("tools"), list): extra_body["realtime_tools"] = session_payload["tools"] if isinstance(session_payload.get("tool_choice"), str | dict): extra_body["realtime_tool_choice"] = session_payload["tool_choice"] metadata = session_payload.get("metadata") if isinstance(metadata, dict): extra_body["realtime_metadata"] = dict(metadata) include = session_payload.get("include") if isinstance(include, list): extra_body["realtime_include"] = list(include) prompt = session_payload.get("prompt") if isinstance(prompt, dict): extra_body["realtime_prompt"] = dict(prompt) input_audio_transcription = self._input_audio_transcription_config(session_payload) if isinstance(input_audio_transcription, dict): extra_body["realtime_input_audio_transcription"] = dict(input_audio_transcription) noise_reduction = session_payload.get("input_audio_noise_reduction") if isinstance(noise_reduction, dict): extra_body["realtime_input_audio_noise_reduction"] = dict(noise_reduction) audio_noise_reduction = audio_input.get("noise_reduction") if isinstance(audio_input, dict) else None if isinstance(audio_noise_reduction, dict): extra_body["realtime_input_audio_noise_reduction"] = dict(audio_noise_reduction) audio = session_payload.get("audio") if isinstance(audio, dict): extra_body["realtime_audio"] = dict(audio) if isinstance(session_payload.get("tracing"), str | dict): extra_body["realtime_tracing"] = session_payload["tracing"] response_format = self._duplex_response_format(self._output_audio_format) extra_body.setdefault("realtime_output_audio_format", self._output_audio_format) overlap_fields = self._realtime_overlap_fields(session_payload) voice = session_payload.get("voice") if not isinstance(voice, str) and isinstance(audio_output, dict): voice = audio_output.get("voice") speed = session_payload.get("speed") if not isinstance(speed, int | float) and isinstance(audio_output, dict): speed = audio_output.get("speed") return { "type": "session.create", "session_id": session_payload.get("session_id") or session_payload.get("id"), "session": { "model": model, "modalities": ( session_payload.get("modalities") or session_payload.get("output_modalities") or ["text", "audio"] ), "instructions": session_payload.get("instructions"), "voice": voice, "ref_audio": session_payload.get("ref_audio"), "response_format": response_format, "temperature": session_payload.get("temperature"), "max_tokens": self.realtime_max_output_tokens( session_payload.get("max_response_output_tokens") or session_payload.get("max_output_tokens") or session_payload.get("max_tokens") ), "speed": speed, "idle_timeout_s": session_payload.get("idle_timeout_s") or 300.0, **overlap_fields, "extra_body": extra_body, }, } def _apply_realtime_session_defaults(self, session_payload: dict[str, object]) -> None: input_format: object = session_payload.get("input_audio_format") audio_config = session_payload.get("audio") if input_format is None and isinstance(audio_config, dict): audio_input = audio_config.get("input") if isinstance(audio_input, dict): input_format = audio_input.get("format") input_format, input_rate = self._parse_realtime_audio_format(input_format) if isinstance(input_format, str) and input_format.lower() in REALTIME_INPUT_AUDIO_FORMATS: self._input_audio_format = input_format output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") output_rate_raw: object | None = None if output_format is None and isinstance(audio_config, dict): audio_output = audio_config.get("output") if isinstance(audio_output, dict): output_format = audio_output.get("format") output_rate_raw = audio_output.get("sample_rate_hz") or audio_output.get("sample_rate") output_format, output_rate = self._parse_realtime_audio_format(output_format) if output_rate is None and isinstance(output_rate_raw, int | float) and output_rate_raw > 0: output_rate = int(output_rate_raw) if isinstance(output_format, str) and output_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: self._output_audio_format = self._realtime_output_format(output_format) sample_rate = session_payload.get("sample_rate_hz") or session_payload.get("sample_rate") if sample_rate is None and isinstance(audio_config, dict): audio_input = audio_config.get("input") if isinstance(audio_input, dict): sample_rate = audio_input.get("sample_rate_hz") or audio_input.get("sample_rate") if sample_rate is None: sample_rate = input_rate if isinstance(sample_rate, int | float) and sample_rate > 0: self._input_sample_rate_hz = int(sample_rate) if isinstance(output_rate, int | float) and output_rate > 0: self._output_sample_rate_hz = int(output_rate) overlap_fields = self._realtime_overlap_fields(session_payload) overlap_silence_rms = overlap_fields.get("overlap_silence_rms") if isinstance(overlap_silence_rms, int | float): self._overlap_silence_rms = max(0.0, float(overlap_silence_rms)) @classmethod def _validate_realtime_session_audio_formats(cls, session_payload: dict[str, object]) -> str | None: audio_config = session_payload.get("audio") input_format: object = session_payload.get("input_audio_format") if input_format is None and isinstance(audio_config, dict): audio_input = audio_config.get("input") if isinstance(audio_input, dict): input_format = audio_input.get("format") parsed_input, _ = cls._parse_realtime_audio_format(input_format) if input_format is not None and not ( isinstance(parsed_input, str) and parsed_input.lower() in REALTIME_INPUT_AUDIO_FORMATS ): return f"Unsupported input_audio_format: {input_format}" output_format: object = session_payload.get("output_audio_format") or session_payload.get("response_format") if output_format is None and isinstance(audio_config, dict): audio_output = audio_config.get("output") if isinstance(audio_output, dict): output_format = audio_output.get("format") parsed_output, _ = cls._parse_realtime_audio_format(output_format) if output_format is not None and not ( isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS ): return f"Unsupported output_audio_format: {output_format}" return None @classmethod def _validate_realtime_response_audio_formats(cls, response_payload: dict[str, object]) -> str | None: output_format: object = response_payload.get("output_audio_format") or response_payload.get("response_format") audio_config = response_payload.get("audio") if output_format is None and isinstance(audio_config, dict): audio_output = audio_config.get("output") if isinstance(audio_output, dict): output_format = audio_output.get("format") parsed_output, _ = cls._parse_realtime_audio_format(output_format) if output_format is not None and not ( isinstance(parsed_output, str) and parsed_output.lower() in REALTIME_OUTPUT_AUDIO_FORMATS ): return f"Unsupported output_audio_format: {output_format}" return None @classmethod def _validate_conversation_item_audio_formats(cls, item: object) -> str | None: if not isinstance(item, dict): return None content = item.get("content") if not isinstance(content, list): return None for part in content: if not isinstance(part, dict): continue if part.get("type") not in {"input_audio", "audio"}: continue raw_format = part.get("format") parsed_format, _ = cls._parse_realtime_audio_format(raw_format) if raw_format is not None and not ( isinstance(parsed_format, str) and parsed_format.lower() in REALTIME_INPUT_AUDIO_FORMATS ): return f"Unsupported input_audio format in conversation.item.create: {raw_format}" return None @staticmethod def _json_safe_realtime_payload(payload: dict[str, object]) -> dict[str, object]: clean: dict[str, object] = {} for key, value in payload.items(): if key == "extra_body": continue if isinstance(value, str | int | float | bool) or value is None: clean[key] = value elif isinstance(value, dict): clean[key] = RealtimeInputTranslator._json_safe_realtime_payload(value) elif isinstance(value, list): clean[key] = [ (RealtimeInputTranslator._json_safe_realtime_payload(item) if isinstance(item, dict) else item) for item in value if isinstance(item, str | int | float | bool | dict) or item is None ] return clean @staticmethod def _parse_realtime_audio_format(raw_format: object) -> tuple[object, int | None]: def normalize_format(fmt: str) -> str: normalized = fmt.lower() if normalized in {"audio/pcm", "pcm"}: return "pcm16" if normalized in {"audio/wav", "wav"}: return "wav" if normalized in {"audio/pcm16", "pcm16", "pcm_s16le", "s16le"}: return "pcm16" if normalized in {"audio/pcm_f32le", "pcm_f32le", "f32le"}: return "pcm_f32le" if normalized in {"audio/g711_ulaw", "g711_ulaw", "g711-ulaw", "ulaw", "mulaw"}: return "g711_ulaw" if normalized in {"audio/g711_alaw", "g711_alaw", "g711-alaw", "alaw"}: return "g711_alaw" return fmt if isinstance(raw_format, str): return normalize_format(raw_format), None if not isinstance(raw_format, dict): return raw_format, None rate = raw_format.get("rate") or raw_format.get("sample_rate_hz") or raw_format.get("sample_rate") sample_rate_hz = int(rate) if isinstance(rate, int | float) and rate > 0 else None fmt = raw_format.get("type") or raw_format.get("format") if not isinstance(fmt, str): return raw_format, sample_rate_hz return normalize_format(fmt), sample_rate_hz @staticmethod def _duplex_response_format(realtime_format: str) -> str: normalized = realtime_format.lower() if normalized in {"pcm16", "pcm_s16le", "s16le"}: return "pcm" if normalized in {"g711_ulaw", "g711_alaw"}: return "pcm" if normalized in {"wav", "pcm"}: return normalized return "wav" @staticmethod def _realtime_output_format(duplex_format: object) -> str: if isinstance(duplex_format, str) and duplex_format.lower() in {"g711_ulaw", "g711_alaw"}: return duplex_format.lower() if isinstance(duplex_format, str) and duplex_format.lower() == "pcm": return "pcm16" return str(duplex_format or "wav") @staticmethod def _is_supported_realtime_input_format(fmt: object) -> bool: return isinstance(fmt, str) and fmt.lower() in REALTIME_INPUT_AUDIO_FORMATS @staticmethod def _input_explicitly_non_speech(event: dict[str, object]) -> bool: for key in ("is_speech", "speech"): value = event.get(key) if isinstance(value, bool): return not value vad = event.get("vad") if isinstance(vad, dict): value = vad.get("is_speech") if isinstance(value, bool): return not value probability = vad.get("speech_probability", vad.get("probability")) if isinstance(probability, int | float): return float(probability) < 0.5 probability = event.get("speech_probability") return isinstance(probability, int | float) and float(probability) < 0.5 def _input_looks_like_speech(self, event: dict[str, object], *, audio: object, fmt: object) -> bool: if RealtimeInputTranslator._input_explicitly_non_speech(event): return False for key in ("is_speech", "speech"): value = event.get(key) if isinstance(value, bool): return value vad = event.get("vad") if isinstance(vad, dict): probability = vad.get("speech_probability", vad.get("probability")) if isinstance(probability, int | float): return float(probability) >= 0.5 probability = event.get("speech_probability") if isinstance(probability, int | float): return float(probability) >= 0.5 if fmt != "pcm_f32le" or not isinstance(audio, str): return True try: raw = base64.b64decode(audio, validate=True) except (binascii.Error, ValueError): return True if len(raw) < 4 or len(raw) % 4 != 0: return True samples = np.frombuffer(raw, dtype=np.float32) if samples.size == 0: return False rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32))))) threshold = event.get("overlap_silence_rms") if not isinstance(threshold, int | float): vad = event.get("vad") if isinstance(vad, dict): threshold = vad.get("silence_rms") silence_rms = float(threshold) if isinstance(threshold, int | float) else self._overlap_silence_rms return rms >= max(0.0, silence_rms) @staticmethod def _copy_realtime_input_hints(source: dict[str, object], target: dict[str, object]) -> None: for key in ( "duration_ms", "audio_duration_ms", "audio_start_ms", "audio_end_ms", "is_speech", "speech", "speech_probability", "vad", "overlap_action", "overlap", "force_barge_in", "force_listen", "text", "transcript", ): if key in source: target[key] = source[key] @classmethod def _realtime_overlap_fields(cls, session_payload: dict[str, object]) -> dict[str, object]: fields: dict[str, object] = {} if isinstance(session_payload.get("overlap_policy"), str): fields["overlap_policy"] = session_payload["overlap_policy"] for key in ("overlap_short_ack_ms", "overlap_barge_in_ms", "overlap_silence_rms"): value = session_payload.get(key) if isinstance(value, int | float): fields[key] = value if isinstance(session_payload.get("playback_commit_policy"), str): fields["playback_commit_policy"] = session_payload["playback_commit_policy"] return fields @staticmethod def _validate_realtime_video_frames(video_frames: object, max_slice_nums: object) -> str | None: """Validate omni-duplex camera frames on input_audio_buffer.append. Wire contract matches the official MiniCPM-o duplex loop: one base base64 JPEG per ~1 s audio chunk, optionally followed by that unit's stacked composite tiling the sub-frames captured inside it (at most 2 images either way). HD slicing (max_slice_nums > 1) is not implemented by the duplex adapter yet and is rejected explicitly rather than silently ignored; note official suggests it for composites (``max_slice_nums=[2, 1]``), so stacked detail is capped at scale_resolution until that lands. """ if max_slice_nums not in (None, 1): return "max_slice_nums > 1 (HD slicing) is not implemented by the duplex Realtime adapter" if not isinstance(video_frames, list): return "video_frames must be a list of base64-encoded images" frames = [frame for frame in video_frames if frame is not None] if len(frames) > 2: return "video_frames carries more than 2 frames for one append; send ~1 frame per 1 s chunk" for frame in frames: if not isinstance(frame, str) or not frame: return "video_frames entries must be non-empty base64 strings" if len(frame) > 4_000_000: return "video_frames entry exceeds 4MB base64; reduce capture resolution or JPEG quality" try: header = base64.b64decode(frame[:64] + "=" * (-len(frame[:64]) % 4)) except (binascii.Error, ValueError): return "video_frames entries must be valid base64" if not (header.startswith(b"\xff\xd8") or header.startswith(b"\x89PNG")): return "video_frames entries must be JPEG or PNG images" return None @staticmethod def _configured_realtime_turn_detection(session_payload: dict[str, object]) -> tuple[str | None, object]: field = "turn_detection" if "turn_detection" in session_payload else None selected = session_payload.get("turn_detection") audio_config = session_payload.get("audio") if isinstance(audio_config, dict): audio_input = audio_config.get("input") if isinstance(audio_input, dict) and "turn_detection" in audio_input: field, selected = "audio.input.turn_detection", audio_input["turn_detection"] return field, selected @classmethod def _validate_realtime_turn_detection(cls, session_payload: dict[str, object]) -> str | None: field, turn_detection = cls._configured_realtime_turn_detection(session_payload) if field is None: if session_payload.get("overlap_policy") == "barge_in_on_speech": return ( "overlap_policy='barge_in_on_speech' requires " "turn_detection.type='server_vad' on the Realtime endpoint" ) return None if turn_detection is not None and not isinstance(turn_detection, dict): return f"{field} must be null or an object with type='server_vad'" if isinstance(turn_detection, dict): if turn_detection.get("type") != "server_vad": return f"{field}.type must be 'server_vad'" threshold = turn_detection.get("threshold", 0.5) if not cls._valid_vad_number(threshold, minimum=SILERO_VAD_MIN_THRESHOLD, strict=True, maximum=1): return f"{field}.threshold must be greater than {SILERO_VAD_MIN_THRESHOLD} and at most 1" for name in ("prefix_padding_ms", "silence_duration_ms", "min_speech_duration_ms"): value = turn_detection.get(name) if value is not None and not cls._valid_vad_number(value, minimum=0): return f"{field}.{name} must be a non-negative number" if turn_detection.get("interrupt_response", True) is not True: return ( f"{field}.interrupt_response=false is unsupported; " "use turn_detection=null for model-owned listen/speak" ) desired_policy = "barge_in_on_speech" if turn_detection is not None else "listen_only" overlap_policy = session_payload.get("overlap_policy") if isinstance(overlap_policy, str) and overlap_policy != desired_policy: return f"overlap_policy={overlap_policy!r} conflicts with turn_detection; expected {desired_policy!r}" return None @staticmethod def _valid_vad_number(value: object, *, minimum: float, maximum: float | None = None, strict: bool = False) -> bool: return ( isinstance(value, int | float) and not isinstance(value, bool) and np.isfinite(float(value)) and (float(value) > minimum if strict else float(value) >= minimum) and (maximum is None or float(value) <= maximum) ) def _prepare_realtime_turn_detection_update(self, session_payload: dict[str, object]) -> None: field, configured = self._configured_realtime_turn_detection(session_payload) if field is None: return if self._pending_turn_detection_update is not None: raise RuntimeError("A Realtime turn-detection update is already pending") turn_detection: dict[str, object] | None = None detector: SileroStreamingVAD | None = None if configured is None: session_payload["overlap_policy"] = "listen_only" else: assert isinstance(configured, dict) turn_detection = { "type": "server_vad", "interrupt_response": True, "threshold": 0.5, "prefix_padding_ms": 300, "silence_duration_ms": 500, "min_speech_duration_ms": 96, **configured, } detector = SileroStreamingVAD( SileroVADConfig( threshold=float(turn_detection["threshold"]), prefix_padding_ms=int(turn_detection["prefix_padding_ms"]), silence_duration_ms=int(turn_detection["silence_duration_ms"]), min_speech_duration_ms=max(32, int(turn_detection["min_speech_duration_ms"])), ) ) session_payload["turn_detection"] = dict(turn_detection) session_payload["overlap_policy"] = "barge_in_on_speech" self._pending_turn_detection_update = (turn_detection, detector, asyncio.Event()) async def wait_for_realtime_turn_detection_update(self) -> None: candidate = self._pending_turn_detection_update if candidate is not None: await candidate[2].wait() def commit_realtime_turn_detection_update(self) -> None: self._resolve_realtime_turn_detection_update(commit=True) def reject_realtime_turn_detection_update(self) -> None: self._resolve_realtime_turn_detection_update(commit=False) def _resolve_realtime_turn_detection_update(self, *, commit: bool) -> None: candidate = self._pending_turn_detection_update if candidate is None: return turn_detection, detector, resolved = candidate discarded = self._server_vad if commit else detector if commit: self._turn_detection, self._server_vad = turn_detection, detector self._pending_turn_detection_update = None resolved.set() reset = getattr(discarded, "reset", None) if callable(reset): reset() def _reset_server_vad(self) -> None: reset = getattr(self._server_vad, "reset", None) if callable(reset): reset() @staticmethod def _input_audio_transcription_config(session_payload: dict[str, object]) -> dict[str, object] | None: transcription = session_payload.get("input_audio_transcription") if isinstance(transcription, dict): return transcription audio_config = session_payload.get("audio") if not isinstance(audio_config, dict): return None audio_input = audio_config.get("input") if not isinstance(audio_input, dict): return None transcription = audio_input.get("transcription") return transcription if isinstance(transcription, dict) else None @staticmethod def realtime_max_output_tokens(value: object) -> int | None: """Normalize Realtime max output tokens. OpenAI Realtime clients commonly send ``"inf"`` for unbounded output. The duplex core represents that as ``None`` because model-specific defaults remain in force. """ if value is None: return None if isinstance(value, str) and value.strip().lower() in {"inf", "infinity", "unlimited"}: return None if isinstance(value, int) and value > 0: return int(value) return None async def _conversation_item_to_duplex(self, event: dict[str, object]) -> dict[str, object] | None: item = event.get("item") if not isinstance(item, dict): return None item = self._normalize_conversation_item(item) previous_item_id = event.get("previous_item_id") if isinstance(previous_item_id, str): item["_previous_item_id"] = previous_item_id item_id = str(item["id"]) item_type = item.get("type") role = item.get("role") if item_type == "function_call_output": call_id = item.get("call_id") output = item.get("output") matching_call = any( known.get("type") == "function_call" and known.get("call_id") == call_id for known in self._conversation_items.values() ) duplicate_output = any( known.get("type") == "function_call_output" and known.get("call_id") == call_id for known in self._conversation_items.values() ) if not isinstance(call_id, str) or not call_id or not matching_call: await self._send_realtime_payload( self._realtime_error_payload( "invalid_function_call_output", "function_call_output requires the call_id of a completed function call", event_id=event.get("event_id"), param="item.call_id", ) ) return None if not isinstance(output, str) or duplicate_output: await self._send_realtime_payload( self._realtime_error_payload( "invalid_function_call_output", ( "function_call_output requires a string output" if not isinstance(output, str) else f"function_call_output already exists for call_id {call_id}" ), event_id=event.get("event_id"), param="item.output", ) ) return None # Do not retain the result until SessionRunner has delivered its # runtime update. On success, the resulting # ``conversation.item.created`` event records it in the shared # input/output protocol state; on failure the client may retry the # same call_id instead of being rejected by provisional history. if item_type != "message" or role in {"assistant", "system"}: return { "type": "turn.signal", "event": "conversation.item.create", "payload": {"item": item}, } self._conversation_items[item_id] = item content = item.get("content") if not isinstance(content, list): return None text_chunks: list[str] = [] audio_events: list[dict[str, object]] = [] for part in content: if not isinstance(part, dict): continue if part.get("type") in {"input_text", "text"} and isinstance(part.get("text"), str): text_chunks.append(str(part["text"])) if part.get("type") in {"input_audio", "audio"}: audio = part.get("audio") or part.get("data") fmt, format_rate = self._parse_realtime_audio_format(part.get("format") or self._input_audio_format) sample_rate_hz = ( part.get("sample_rate_hz") or part.get("sample_rate") or format_rate or self._input_sample_rate_hz ) if not self._is_supported_realtime_input_format(fmt): continue try: audio, fmt, sample_rate_hz = convert_input_audio_with_rate( audio, fmt, sample_rate_hz=sample_rate_hz if isinstance(sample_rate_hz, int | float) else None, ) except ValueError as exc: await self._send_realtime_payload( self._realtime_error_payload( "bad_event", str(exc), event_id=event.get("event_id"), param="sample_rate_hz", ) ) return None if not isinstance(audio, str) or not audio: continue speech_hints = dict(event) speech_hints.update(part) if not self._input_looks_like_speech(speech_hints, audio=audio, fmt=fmt): continue self._input_speech_started = True payload = { "type": "input_audio_buffer.append", "audio": audio, "format": fmt, "sample_rate_hz": sample_rate_hz, } self._copy_realtime_input_hints(part, payload) self._copy_realtime_input_hints(event, payload) audio_events.append(payload) if audio_events: self._input_audio_buffer_has_audio = True transcript = self._input_transcript_from_item(item) for extra_event in audio_events[1:]: self._pending_outbound.put_nowait(extra_event) commit_payload: dict[str, object] = { "type": "input_audio_buffer.commit", "final": True, "realtime_item_id": item_id, "response_create": False, } if transcript: commit_payload["transcript"] = transcript self._pending_outbound.put_nowait(commit_payload) return audio_events[0] if not text_chunks: return None text_item = dict(item) text_item["status"] = "completed" return { "type": "turn.signal", "event": "conversation.item.create", "payload": {"item": text_item}, } def _remember_input_transcript_hint(self, event: dict[str, object]) -> None: transcript = event.get("transcript") if not isinstance(transcript, str): transcript = event.get("text") if isinstance(event.get("text"), str) else None if not isinstance(transcript, str): hints = event.get("hints") if isinstance(hints, dict): transcript = hints.get("transcript") if not isinstance(transcript, str): transcript = hints.get("text") if isinstance(hints.get("text"), str) else None if isinstance(transcript, str) and transcript: if ( self._input_audio_buffer_transcript_parts and self._input_audio_buffer_transcript_parts[-1] == transcript ): return self._input_audio_buffer_transcript_parts.append(transcript) def _consume_input_transcript_hint(self) -> str: transcript = "".join(self._input_audio_buffer_transcript_parts).strip() self._input_audio_buffer_transcript_parts.clear() return transcript @staticmethod def _input_transcript_from_item(item: dict[str, object]) -> str: content = item.get("content") if not isinstance(content, list): return "" parts: list[str] = [] for part in content: if not isinstance(part, dict): continue for key in ("transcript", "text"): value = part.get(key) if isinstance(value, str) and value: if part.get("type") in {"input_audio", "audio", "audio_transcript", "transcript"}: parts.append(value) break return "".join(parts).strip() @staticmethod def _input_audio_transcription_completed_event( item_id: str, item: dict[str, object], ) -> dict[str, object] | None: content = item.get("content") if not isinstance(content, list): return None transcript_parts: list[str] = [] for index, part in enumerate(content): if not isinstance(part, dict): continue if part.get("type") != "input_audio": continue transcript = part.get("transcript") if isinstance(transcript, str) and transcript: transcript_parts.append(transcript) return { "type": "conversation.item.input_audio_transcription.completed", "item_id": item_id, "content_index": index, "transcript": "".join(transcript_parts), } return None def _pop_pending_commit_item_id(self) -> str: try: return self._pending_commit_item_ids.get_nowait() except asyncio.QueueEmpty: return f"item_{uuid4().hex}" async def _emit_input_speech_started(self, event: dict[str, object]) -> None: if self._input_speech_started: return self._input_speech_started = True if self._active_input_item_id is None: self._active_input_item_id = f"item_{uuid4().hex}" audio_start_ms = event.get("audio_start_ms", 0) await self._send_realtime_payload( { "type": "input_audio_buffer.speech_started", "audio_start_ms": int(audio_start_ms) if isinstance(audio_start_ms, int | float) else 0, "item_id": self._active_input_item_id, } ) async def _emit_input_speech_stopped(self, event: dict[str, object], *, item_id: str) -> None: if not self._input_speech_started: return self._input_speech_started = False audio_end_ms = event.get("audio_end_ms", event.get("audio_ms", 0)) await self._send_realtime_payload( { "type": "input_audio_buffer.speech_stopped", "audio_end_ms": int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0, "item_id": item_id, } )