# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import asyncio import base64 import binascii import inspect import json from collections.abc import Mapping, MutableMapping from contextlib import suppress from dataclasses import dataclass, replace from typing import TYPE_CHECKING import numpy as np from fastapi import WebSocket from vllm.logger import init_logger from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig from vllm_omni.engine.duplex.messages import DuplexFence, DuplexSessionLifecycleMessage from vllm_omni.engine.duplex.runtime import duplex_resource_request_id from vllm_omni.entrypoints.duplex.capability import ( should_enable_duplex_endpoint, ) from vllm_omni.entrypoints.duplex.chat_fallback import ( ChatFallbackProjectorMixin, ) from vllm_omni.entrypoints.duplex.protocol import ( NATIVE_DUPLEX_KEY, DuplexCapabilities, DuplexCommittedInput, DuplexOverlapPolicy, DuplexPlaybackCommitPolicy, DuplexSession, DuplexSessionConfig, DuplexSessionRegistry, DuplexSessionState, DuplexTurnController, ResponseCreateOptions, native_duplex_opt_in, normalize_native_duplex_key, ) from vllm_omni.entrypoints.duplex.realtime_session import ( REALTIME_OUTPUT_AUDIO_FORMATS, NativeRealtimeSessionProtocol, ) from vllm_omni.entrypoints.duplex.runtime_adapter import ( ServingRuntimeAdapter, ServingRuntimeConfigError, ServingRuntimeSessionState, TurnBasedServingSessionState, load_serving_runtime_adapter, validate_serving_runtime_adapter, ) from vllm_omni.entrypoints.duplex.runtime_bridge import ( NativeRuntimeBridgeMixin, ) from vllm_omni.entrypoints.duplex.server_vad import ( SILERO_VAD_DEFAULT_MIN_SPEECH_DURATION_MS, SILERO_VAD_MIN_THRESHOLD, ServerVADPipeline, SileroVADBackendProvider, SpeechDetectorBackendProvider, parse_session_turn_detection, ) from vllm_omni.entrypoints.duplex.session_attachment import ( DuplexJournalGapError, DuplexSessionAttachmentRegistry, InvalidResumeTokenError, ) from vllm_omni.entrypoints.duplex.session_runner import ( DuplexSessionRunnerMixin, ) from vllm_omni.entrypoints.duplex.websocket import ( DOMAIN_TERMINAL_EVENTS, DuplexSessionTasks, DuplexWebSocketActor, ) from vllm_omni.metrics.realtime import RealtimeVADMetrics if TYPE_CHECKING: from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat logger = init_logger(__name__) __all__ = ["OmniDuplexSessionHandler", "should_enable_duplex_endpoint"] _DEFAULT_CONFIG_TIMEOUT_S = 10.0 _DEFAULT_IDLE_TIMEOUT_S = 300.0 @dataclass(frozen=True) class _DuplexSessionHandshake: session: DuplexSession resumed: bool = False attachment_generation: int | None = None event_id: str | None = None class OmniDuplexSessionHandler( DuplexSessionRunnerMixin, NativeRuntimeBridgeMixin, ChatFallbackProjectorMixin, ): """WebSocket handler for RFC-style full-duplex session control. This owns the serving-side session actor, ordered inbound mailbox, barge-in epoch, turn controller, and playback commit state. Generic sessions can still fall back to chat requests, while MiniCPM-o 4.5 native sessions route audio appends through scheduler data-plane stage requests. It deliberately does not claim core persistent KV lease support. """ def __init__( self, *, chat_service: OmniOpenAIServingChat, served_model_name: str | None = None, config_timeout_s: float = _DEFAULT_CONFIG_TIMEOUT_S, idle_timeout_s: float = _DEFAULT_IDLE_TIMEOUT_S, log_stats: bool = True, duplex_session_config: DuplexSessionRuntimeConfig | None = None, serving_runtime_adapter: ServingRuntimeAdapter | None = None, serving_runtime_adapter_path: str | None = None, server_vad_backend_provider: SpeechDetectorBackendProvider | None = None, ) -> None: self._chat_service = chat_service self._config_timeout_s = config_timeout_s self._idle_timeout_s = idle_timeout_s self._duplex_session_config = duplex_session_config or DuplexSessionRuntimeConfig() self._server_vad_backend_provider = server_vad_backend_provider or SileroVADBackendProvider( model_path=self._duplex_session_config.server_vad_model_path, ) self._server_vad_pipelines: dict[str, ServerVADPipeline] = {} self._realtime_vad_metrics = RealtimeVADMetrics( served_model_name or chat_service.model_config.model, log_stats=log_stats, ) self._registry = DuplexSessionRegistry( DuplexCapabilities( supports_model_native_turn_policy=False, supports_input_append=False, supports_replace_latest_chunk=True, supports_reencode_context=True, supports_turn_commit_only=True, supports_kv_lease=False, ) ) self._turn_controller = DuplexTurnController() adapter_path = serving_runtime_adapter_path or getattr( chat_service, "duplex_serving_adapter_path", None, ) self._serving_runtime_adapter: ServingRuntimeAdapter | None if serving_runtime_adapter is not None: self._serving_runtime_adapter = validate_serving_runtime_adapter(serving_runtime_adapter) elif isinstance(adapter_path, str) and adapter_path: self._serving_runtime_adapter = load_serving_runtime_adapter( adapter_path, self._encode_native_data_plane_audio, ) else: self._serving_runtime_adapter = None self._serving_session_states: MutableMapping[str, ServingRuntimeSessionState] = ( self._serving_runtime_adapter.session_states if self._serving_runtime_adapter is not None else {} ) self._session_tasks: dict[str, DuplexSessionTasks] = {} self._realtime_protocols: dict[str, NativeRealtimeSessionProtocol] = {} self._lease_generations: dict[str, int] = {} self._resync_required_sessions: set[str] = set() self._lifecycle_queue = getattr(self._chat_service.engine_client, "duplex_lifecycle_events", None) self._lifecycle_task: asyncio.Task[None] | None = None self._attachment_registry = DuplexSessionAttachmentRegistry( replay_ttl_s=self._duplex_session_config.resume_replay_ttl_s, replay_max_bytes_per_session=self._duplex_session_config.resume_replay_max_bytes_per_session, disconnect_grace_s=self._duplex_session_config.disconnect_grace_s, ) async def handle_realtime_session(self, websocket: WebSocket) -> None: await self.handle_session( websocket, realtime_protocol=NativeRealtimeSessionProtocol(websocket.query_params), ) def _ensure_lifecycle_listener(self) -> None: if not isinstance(self._lifecycle_queue, asyncio.Queue): return if self._lifecycle_task is None or self._lifecycle_task.done(): self._lifecycle_task = asyncio.create_task( self._run_lifecycle_listener(), name="duplex-serving-lifecycle", ) async def _run_lifecycle_listener(self) -> None: queue = self._lifecycle_queue if not isinstance(queue, asyncio.Queue): return try: while True: message = await queue.get() try: if isinstance(message, DuplexSessionLifecycleMessage): await self._apply_runtime_lifecycle(message) finally: queue.task_done() if self._registry.active_count() == 0: return finally: if self._lifecycle_task is asyncio.current_task(): self._lifecycle_task = None async def _apply_runtime_lifecycle(self, message: DuplexSessionLifecycleMessage) -> None: session = self._registry.get(message.session_id) if session is None: return fence = message.fence if ( fence.session_id != session.session_id or fence.incarnation != session.incarnation or fence.epoch < session.epoch ): return current_generation = self._lease_generations.get(session.session_id, 0) if message.lease_generation < current_generation: return protocol = self._realtime_protocols.get(session.session_id) expired_payload: dict[str, object] = { "type": "session.expired", "session_id": session.session_id, "incarnation": session.incarnation, "reason": message.reason, } if protocol is not None: expired_payload = protocol.encode_outbound_event(expired_payload)[0] with suppress(Exception): await self._attachment_registry.send_event( session.session_id, expired_payload, journal=False, ) tasks = self._session_tasks.pop(session.session_id, None) if tasks is not None: await tasks.cancel_append_tasks() active_response_task = tasks.active_response_task if active_response_task is not None and not active_response_task.done(): active_response_task.cancel() with suppress(asyncio.CancelledError): await active_response_task native = self._serving_session_states.get(session.session_id) if native is not None and native.data_plane_task is not None: data_plane_task = native.data_plane_task native.data_plane_task = None data_plane_task.cancel() with suppress(asyncio.CancelledError): await data_plane_task if session.active_response_id is not None: session.end_response(commit_text=False) self._cleanup_duplex_session_state(session) self._registry.close(session.session_id) self._realtime_protocols.pop(session.session_id, None) self._lease_generations.pop(session.session_id, None) self._resync_required_sessions.discard(session.session_id) attachment = await self._attachment_registry.close(session.session_id) if attachment is not None: with suppress(Exception): await attachment.close("session_expired") def _stop_lifecycle_listener_if_idle(self) -> None: task = self._lifecycle_task if self._registry.active_count() != 0 or task is None or task.done(): return if task is not asyncio.current_task(): task.cancel() self._lifecycle_task = None @staticmethod def _native_audio_payload_size_bytes(payload: Mapping[str, object]) -> int: audio = payload.get("audio") or payload.get("data") if not isinstance(audio, str): return 0 try: return len(base64.b64decode(audio, validate=True)) except (ValueError, binascii.Error): return 0 async def _apply_outbound_session_event( self, payload: dict[str, object], *, session: DuplexSession | None, actor: DuplexWebSocketActor, native: ServingRuntimeSessionState, realtime_protocol: NativeRealtimeSessionProtocol | None, ) -> tuple[bool, dict[str, object] | None]: """Apply domain transitions before an event is queued for transport.""" payload_type = payload.get("type") is_terminal = payload_type in DOMAIN_TERMINAL_EVENTS if is_terminal and session is not None: payload_epoch = payload.get("epoch") if isinstance(payload_epoch, int) and payload_epoch != session.epoch: return False, None if payload_type in {"response.done", "response.listen"} and ( actor.closing or session.state == DuplexSessionState.CLOSED ): return False, None elif actor._is_stale_model_output(payload): return False, None if payload_type == "session.closed": actor.close_reason = actor.close_reason or str(payload.get("reason") or "closed") if session is not None: session.mark_closing() if not is_terminal or session is None: return True, None terminal_status = payload.get("status") terminal_status_details = payload.get("status_details") if terminal_status is None and isinstance(terminal_status_details, dict): terminal_status = terminal_status_details.get("type") response_terminal = payload_type == "response.done" or ( payload_type == "response.listen" and session.active_response_id is not None ) can_promote_overlap = response_terminal and terminal_status not in {"cancelled", "failed"} deferred_overlap_payload: dict[str, object] | None = None continuous_input_crosses_terminal = ( can_promote_overlap and realtime_protocol is not None and self._session_auto_responds(session) and native.input_since_commit and not native.deferred_response_create ) if continuous_input_crosses_terminal: # Auto-response is a model-owned continuous stream. A response # terminal advances the model turn without requiring the browser # to close its Realtime input item. Keep any partial model unit so # later PCM can complete it. Serving does not create or promote a # new model generation here; Stage0 advances generation identity # only after the model's terminal token. session.reset_overlap_speech() return True, None realtime_input_still_open = ( can_promote_overlap and realtime_protocol is not None and native.input_since_commit and not native.deferred_response_create ) if realtime_input_still_open: # response.done only closes the assistant response. It does not # close the current Realtime input item, and it does not reserve # or advance a model generation. session.reset_overlap_speech() return True, None if can_promote_overlap and session.overlap_speech_ms > 0: has_deferred_overlap = native.audio_buffer.has_pending() or native.committed_audio_payload is not None should_promote_overlap = ( session.state == DuplexSessionState.OPEN and self._uses_native_input_append(session) and has_deferred_overlap and session.overlap_speech_ms > session.config.overlap_short_ack_ms ) if should_promote_overlap: flushed_reserved_bytes = native.audio_buffer.pending_byte_count deferred_overlap_payload = native.audio_buffer.flush( chunk_period_ms=session.capabilities.chunk_period_ms or 1000 ) if native.committed_audio_payload is not None: if deferred_overlap_payload is not None: deferred_overlap_payload = self._merge_native_audio_payloads( native.committed_audio_payload, deferred_overlap_payload, ) else: deferred_overlap_payload = native.committed_audio_payload if self._session_auto_responds(session) and deferred_overlap_payload is not None: deferred_overlap_payload = dict(deferred_overlap_payload) deferred_overlap_payload["force_listen"] = False if deferred_overlap_payload is not None: native.retain_committed_audio( deferred_overlap_payload, operation_id=native.committed_audio_operation_id, reserved_bytes=flushed_reserved_bytes, ) native.input_since_commit = deferred_overlap_payload is not None if realtime_protocol is not None: native.deferred_response_create = True native.deferred_precreate_response = False deferred_overlap_payload = None else: had_pending_overlap_audio = native.audio_buffer.has_pending() native.audio_buffer.clear() native.input_since_commit = False native.speech_since_commit = False if had_pending_overlap_audio and realtime_protocol is not None: await realtime_protocol.discard_pending_input_audio(audio_end_ms=session.overlap_speech_ms) if payload_type in {"audio.cancelled", "input.cancelled", "session.closed"}: session.release_input_bytes(native.clear_committed_audio()) session.reset_overlap_speech() if can_promote_overlap and native.deferred_response_create and native.committed_audio_payload is not None: deferred_overlap_payload = native.committed_audio_payload native.deferred_response_create = False native.input_since_commit = False native.speech_since_commit = False return True, deferred_overlap_payload @staticmethod def _advance_barge_in_epoch(session: DuplexSession) -> tuple[int, dict[str, int]]: old_playback = session.playback.as_dict() new_epoch = session.barge_in() session.clear_playback_cursor() return new_epoch, old_playback @staticmethod def _commit_played_response_history( session: DuplexSession, response_id: str | None, committed_ms: int, ) -> None: if not response_id or committed_ms < 0: return session.truncate_history_item( f"item_{response_id}", audio_end_ms=committed_ms, playback=session.playback_for_response(response_id), ) @staticmethod def _should_commit_response_to_history(session: DuplexSession, response_id: str | None) -> bool: if response_id is not None and response_id != session.active_response_id: return True mode = session.response_config.extra_body.get("realtime_response_conversation") return not isinstance(mode, str) or mode.strip().lower() != "none" def _response_created_payload( self, session: DuplexSession, response_id: str, *, epoch: int, request_id: str | None = None, ) -> dict[str, object]: response_config = session.response_config payload: dict[str, object] = { "type": "response.created", "session_id": session.session_id, "response_id": response_id, "epoch": epoch, "modalities": list(response_config.modalities), } if request_id is not None: payload["request_id"] = request_id metadata = response_config.extra_body.get("realtime_response_metadata") if not isinstance(metadata, dict): metadata = response_config.extra_body.get("realtime_metadata") if isinstance(metadata, dict): payload["metadata"] = dict(metadata) conversation = response_config.extra_body.get("realtime_response_conversation") if isinstance(conversation, str): payload["conversation"] = conversation prompt = response_config.extra_body.get("realtime_response_prompt") if isinstance(prompt, dict): payload["prompt"] = dict(prompt) return payload def _overlap_decision( self, session: DuplexSession, event: dict[str, object], payload: dict[str, object], ) -> dict[str, object]: """Classify input that arrives while assistant audio is active. This is a serving-side policy. The model still owns listen/speak decisions for normal chunks; overlap policy only decides whether the current assistant response should be interrupted before the new audio is appended. """ duration_ms = self._input_audio_duration_ms(event, payload) is_speech = self._input_looks_like_speech(event, payload, session=session) if not session.capabilities.supports_barge_in and self._event_requests_barge_in(event): return self._defer_unsupported_barge_in(session, duration_ms=duration_ms, is_speech=is_speech) explicit = event.get("overlap_action") or event.get("overlap") if isinstance(explicit, str): normalized = explicit.strip().lower() if normalized in {"barge_in", "interrupt", "cancel"}: return { "action": "barge_in", "reason": "client_overlap_action", "duration_ms": duration_ms, "buffer_audio": True, } if normalized in {"listen", "continue", "continue_output", "ack"}: session.reset_overlap_speech() return { "action": "listen", "reason": "client_overlap_action", "duration_ms": duration_ms, "buffer_audio": ( normalized == "listen" and is_speech and duration_ms > session.config.overlap_short_ack_ms ), "defer_runtime_append": True, } if normalized in {"drop", "ignore", "silence"}: session.reset_overlap_speech() return { "action": "drop", "reason": "client_overlap_action", "duration_ms": duration_ms, "buffer_audio": False, } if bool(event.get("force_barge_in", False)): return { "action": "barge_in", "reason": "client_force_barge_in", "duration_ms": duration_ms, "buffer_audio": True, } if self._session_auto_responds(session): # Full-duplex input remains model-owned while output is active. Feed # complete model units into the existing Stage0 stream immediately; # playback only controls history ACKs, not model admission. if is_speech: session.accumulate_overlap_speech(duration_ms) vad_speech_started = self._vad_speech_started(event, payload) if ( self._uses_native_input_append(session) and session.capabilities.supports_barge_in and is_speech and session.config.overlap_policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value and vad_speech_started is not False ): return { "action": "barge_in", "reason": ("server_vad_speech_started" if vad_speech_started is True else "barge_in_on_speech"), "cancel_reason": "turn_detected" if vad_speech_started is True else "barge_in", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, } return { "action": "listen", "reason": "auto_response_continuous", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, "defer_runtime_append": False, "force_listen": event.get("force_listen") is True or payload.get("force_listen") is True, "preserve_realtime_input": True, } if bool(event.get("force_listen", False)): session.reset_overlap_speech() return { "action": "listen", "reason": "client_force_listen", "duration_ms": duration_ms, "buffer_audio": is_speech, "defer_runtime_append": True, } policy = session.config.overlap_policy if not is_speech: if session.overlap_speech_ms <= 0: session.reset_overlap_speech() return { "action": "drop", "reason": "silence_or_noise", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": False, } if self._is_short_ack_transcript_hint(event, payload): session.reset_overlap_speech() return { "action": "listen", "reason": "short_ack_transcript", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": False, "defer_runtime_append": True, } if policy == DuplexOverlapPolicy.LISTEN_ONLY.value: session.accumulate_overlap_speech(duration_ms) return { "action": "listen", "reason": "policy_listen_only", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, "defer_runtime_append": True, } if policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value and not session.capabilities.supports_barge_in: return self._defer_unsupported_barge_in(session, duration_ms=duration_ms, is_speech=True) session.accumulate_overlap_speech(duration_ms) if policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value: vad_speech_started = self._vad_speech_started(event, payload) if vad_speech_started is False: return { "action": "listen", "reason": "server_vad_utterance_active", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, "defer_runtime_append": False, "force_listen": True, "preserve_realtime_input": True, } return { "action": "barge_in", "reason": ("server_vad_speech_started" if vad_speech_started is True else "policy_barge_in_on_speech"), "cancel_reason": "turn_detected" if vad_speech_started is True else "barge_in", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, } if ( duration_ms <= session.config.overlap_short_ack_ms and session.overlap_speech_ms <= session.config.overlap_short_ack_ms ): return { "action": "listen", "reason": "short_ack", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, "defer_runtime_append": True, } if session.overlap_speech_ms >= session.config.overlap_barge_in_ms: if not session.capabilities.supports_barge_in: return { "action": "listen", "reason": "barge_in_unsupported", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, "defer_runtime_append": True, } return { "action": "barge_in", "reason": "long_overlap_speech", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, } return { "action": "listen", "reason": "accumulating_overlap_speech", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": True, "defer_runtime_append": True, } @staticmethod def _vad_speech_started( event: Mapping[str, object], payload: Mapping[str, object], ) -> bool | None: for source in (event, payload): vad = source.get("vad") if isinstance(vad, Mapping) and isinstance(vad.get("speech_started"), bool): return bool(vad["speech_started"]) return None @staticmethod def _event_requests_barge_in(event: Mapping[str, object]) -> bool: if event.get("force_barge_in") is True: return True explicit = event.get("overlap_action") or event.get("overlap") return isinstance(explicit, str) and explicit.strip().lower() in { "barge_in", "interrupt", "cancel", } @staticmethod def _defer_unsupported_barge_in( session: DuplexSession, *, duration_ms: int, is_speech: bool, ) -> dict[str, object]: if is_speech: session.accumulate_overlap_speech(duration_ms) return { "action": "listen", "reason": "barge_in_unsupported", "duration_ms": duration_ms, "overlap_speech_ms": session.overlap_speech_ms, "buffer_audio": is_speech, "defer_runtime_append": True, } @staticmethod def _barge_in_unsupported_error(session: DuplexSession) -> dict[str, object]: return { "type": "error", "session_id": session.session_id, "code": "barge_in_unsupported", "error": "Barge-in is not supported by this duplex model", } @staticmethod def _is_short_ack_transcript_hint(event: dict[str, object], payload: dict[str, object]) -> bool: raw_text = event.get("transcript") or event.get("text") or payload.get("transcript") or payload.get("text") if not isinstance(raw_text, str): return False normalized = raw_text.strip().lower() if not normalized: return False compact = "".join(ch for ch in normalized if ch.isalnum() or "\u4e00" <= ch <= "\u9fff") if compact in { "嗯", "嗯嗯", "对", "对的", "好", "好的", "继续", "继续说", "可以", "是的", "yes", "yeah", "yep", "ok", "okay", "continue", "goon", "right", }: return True return normalized in {"go on", "keep going", "please continue"} @staticmethod def _input_audio_duration_ms(event: dict[str, object], payload: dict[str, object]) -> int: for key in ("duration_ms", "audio_duration_ms"): value = event.get(key) if isinstance(value, int | float): return max(0, int(value)) fmt = payload.get("format") sample_rate_hz = payload.get("sample_rate_hz") audio = payload.get("audio") if fmt == "pcm_f32le" and isinstance(sample_rate_hz, int) and sample_rate_hz > 0 and isinstance(audio, str): try: raw = base64.b64decode(audio, validate=True) except (binascii.Error, ValueError): return 0 return int((len(raw) // 4) * 1000 / sample_rate_hz) return 0 @staticmethod def _merge_native_audio_payloads( first: dict[str, object], second: dict[str, object], ) -> dict[str, object]: if first.get("format") != "pcm_f32le" or second.get("format") != "pcm_f32le": return second first_rate = first.get("sample_rate_hz") second_rate = second.get("sample_rate_hz") if not isinstance(first_rate, int) or not isinstance(second_rate, int) or first_rate != second_rate: return second first_audio = first.get("audio") second_audio = second.get("audio") if not isinstance(first_audio, str) or not isinstance(second_audio, str): return second try: first_raw = base64.b64decode(first_audio, validate=True) second_raw = base64.b64decode(second_audio, validate=True) except (binascii.Error, ValueError): return second merged = dict(second) merged["audio"] = base64.b64encode(first_raw + second_raw).decode("ascii") merged["sample_rate_hz"] = first_rate merged_frames = [ frame for source in (first.get("video_frames"), second.get("video_frames")) if isinstance(source, list) for frame in source if isinstance(frame, str) and frame ] if merged_frames: merged["video_frames"] = merged_frames else: merged.pop("video_frames", None) merged["force_listen"] = bool(first.get("force_listen", False)) or bool(second.get("force_listen", False)) merged.pop("force_speak", None) merged["is_speech"] = bool(first.get("is_speech", False)) or bool(second.get("is_speech", False)) return merged @classmethod def _should_force_listen_for_short_commit( cls, session: DuplexSession, event: dict[str, object], payload: dict[str, object], ) -> bool: """Keep very short committed Realtime chunks in listen mode. Realtime VAD can emit a commit for a short pause even when the user has not actually yielded the turn. For MiniCPM-o native duplex, make that policy explicit by steering the scheduler path to the model listen token instead of letting a sub-second chunk start a response. """ if event.get("force_listen") is True or payload.get("force_listen") is True: return True if event.get("force_barge_in") is True: return False if event.get("response_create") is not True: return False duration_ms = cls._input_audio_duration_ms(event, payload) return 0 < duration_ms <= session.config.overlap_short_ack_ms def _create_runtime_session_state(self) -> ServingRuntimeSessionState: adapter = self._serving_runtime_adapter if adapter is not None: return adapter.create_session_state() return TurnBasedServingSessionState() def _runtime_session_state(self, session: DuplexSession) -> ServingRuntimeSessionState: adapter = self._serving_runtime_adapter if adapter is not None: return adapter.session_state(session.session_id) state = self._serving_session_states.get(session.session_id) if state is None: state = TurnBasedServingSessionState() self._serving_session_states[session.session_id] = state return state def _require_serving_runtime_adapter(self) -> ServingRuntimeAdapter: adapter = self._serving_runtime_adapter if adapter is None: raise RuntimeError("Model-native duplex runtime is not configured for this session") return adapter def _should_force_listen_for_auto_response_overlap( self, session: DuplexSession, event: dict[str, object], payload: dict[str, object], ) -> bool: if not self._session_auto_responds(session): return False if event.get("force_barge_in") is True: return False return event.get("force_listen") is True or payload.get("force_listen") is True @staticmethod def _assistant_playback_active(session: DuplexSession) -> bool: return ( session.config.playback_commit_policy == DuplexPlaybackCommitPolicy.ACK_ONLY.value and session.playback.sent_ms > session.playback.committed_ms ) @staticmethod def _input_looks_like_speech( event: dict[str, object], payload: dict[str, object], *, session: DuplexSession, ) -> bool: # Prefer serving-derived fields in ``payload`` over optional client # hints retained in ``event``. In server-VAD mode the pipeline, rather # than the client or the RMS fallback below, owns speech classification. for source in (payload, event): for key in ("is_speech", "speech"): value = source.get(key) if isinstance(value, bool): return value vad = source.get("vad") if isinstance(vad, dict): value = vad.get("is_speech") if isinstance(value, bool): return value probability = vad.get("speech_probability", vad.get("probability")) if isinstance(probability, int | float): return float(probability) >= 0.5 probability = source.get("speech_probability") if isinstance(probability, int | float): return float(probability) >= 0.5 fmt = payload.get("format") audio = payload.get("audio") if fmt in {"pcm_f32le", "pcm16"} and isinstance(audio, str): try: raw = base64.b64decode(audio, validate=True) except (binascii.Error, ValueError): return True if fmt == "pcm_f32le": if len(raw) < 4 or len(raw) % 4 != 0: return True samples = np.frombuffer(raw, dtype=np.float32) else: if len(raw) < 2 or len(raw) % 2 != 0: return True samples = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0 if samples.size == 0: return False rms = float(np.sqrt(np.mean(np.square(samples.astype(np.float32))))) return rms >= session.config.overlap_silence_rms return True async def _emit_overlap_decision( self, send_json, session: DuplexSession, decision: dict[str, object], ) -> None: await send_json( { "type": "overlap.decision", "session_id": session.session_id, "epoch": session.epoch, "policy": session.config.overlap_policy, **decision, } ) async def _open_session( self, websocket: WebSocket, send_json, *, realtime_protocol: NativeRealtimeSessionProtocol | None = None, attachment_send=None, attachment_close=None, ) -> _DuplexSessionHandshake | None: raw = await self._receive_text( websocket, self._config_timeout_s, realtime_protocol=realtime_protocol, ) if raw is None: await send_json({"type": "error", "error": "Timeout waiting for session.create", "code": "config_timeout"}) return None try: event = json.loads(raw) except json.JSONDecodeError: await send_json({"type": "error", "error": "Invalid JSON in session.create", "code": "invalid_json"}) return None if isinstance(event, dict) and event.get("type") == "session.resume": if realtime_protocol is None: await send_json( { "type": "error", "error": "session.resume requires the Realtime protocol", "code": "unsupported_session_resume", } ) return None return await self._resume_session_handshake( event, send_json=send_json, realtime_protocol=realtime_protocol, attachment_send=attachment_send, attachment_close=attachment_close, ) if not isinstance(event, dict) or event.get("type") not in {"session.create", "open_session", "session.config"}: await send_json( { "type": "error", "error": f"Expected session.create, got: {event.get('type') if isinstance(event, dict) else None}", "code": "bad_event", } ) return None try: config = DuplexSessionConfig.from_event(event) except ValueError as exc: error: dict[str, object] = { "type": "error", "error": str(exc), "code": "invalid_request_error", } event_id = event.get("_realtime_event_id") if isinstance(event_id, str) and event_id: error["event_id"] = event_id await send_json(error) return None if config.idle_timeout_s == _DEFAULT_IDLE_TIMEOUT_S: config.idle_timeout_s = self._idle_timeout_s runtime_adapter = self._serving_runtime_adapter if self._uses_serving_runtime_adapter(config) else None runtime_config: dict[str, object] = {} if runtime_adapter is not None: try: runtime_config = await runtime_adapter.prepare_runtime_config( config, model_config=getattr(self._chat_service, "model_config", None), ) except ServingRuntimeConfigError as exc: await send_json({"type": "error", "error": str(exc), "code": exc.code}) return None except ValueError as exc: await send_json({"type": "error", "error": str(exc), "code": "unsupported_ref_audio_path"}) return None session_id = event.get("session_id") if isinstance(event.get("session_id"), str) else None session = self._registry.create(config=config, session_id=session_id) if runtime_adapter is not None: session.replace_capabilities( runtime_adapter.capabilities(max_sessions=self._duplex_session_config.max_sessions) ) session.replace_runtime_config(runtime_config) session_payload = event.get("session") self._resolve_server_vad_defaults( session, config, session_payload if isinstance(session_payload, dict) else event ) event_id = event.get("_realtime_event_id") return _DuplexSessionHandshake( session=session, event_id=event_id if isinstance(event_id, str) and event_id else None, ) async def _resume_session_handshake( self, event: dict[str, object], *, send_json, realtime_protocol: NativeRealtimeSessionProtocol, attachment_send, attachment_close, ) -> _DuplexSessionHandshake | None: session_id = event.get("session_id") incarnation = event.get("incarnation") resume_token = event.get("resume_token") last_received = event.get("last_received_server_event_seq", 0) if ( not isinstance(session_id, str) or not session_id or not isinstance(incarnation, int) or incarnation < 0 or not isinstance(resume_token, str) or not resume_token or not isinstance(last_received, int) or last_received < 0 ): await send_json( { "type": "error", "error": ( "session.resume requires session_id, incarnation, resume_token, " "and a non-negative event sequence" ), "code": "invalid_session_resume", } ) return None session = self._registry.get(session_id) if session is None or session.state != DuplexSessionState.OPEN: await send_json( { "type": "error", "error": f"Unknown or expired duplex session: {session_id}", "code": "session_resume_expired", } ) return None if not session.capabilities.supports_session_resume: await send_json( { "type": "error", "error": f"Session does not support resume: {session_id}", "code": "unsupported_session_resume", } ) return None try: await self._attachment_registry.authenticate_resume( session_id, incarnation=incarnation, resume_token=resume_token, last_received_server_event_seq=last_received, ) except InvalidResumeTokenError: await send_json( { "type": "error", "error": "Invalid duplex session resume token", "code": "invalid_resume_token", } ) return None except DuplexJournalGapError: await send_json( { "type": "session.resync_required", "session_id": session_id, "reason": "journal_gap", } ) return None except (KeyError, ValueError) as exc: await send_json( { "type": "error", "error": str(exc), "code": "session_resume_conflict", } ) return None resume_runtime = getattr(self._chat_service.engine_client, "resume_duplex_session_async", None) if not callable(resume_runtime): await send_json( { "type": "error", "error": "Duplex runtime does not expose session resume control", "code": "runtime_resume_unsupported", } ) return None expected_generation = self._lease_generations.get(session_id, 0) try: runtime_result = await resume_runtime( session_id, fence=DuplexFence( session.session_id, epoch=session.epoch, turn_id=session.turn_id, incarnation=session.incarnation, ), expected_lease_generation=expected_generation, ) lease_generation = self._runtime_lease_generation(runtime_result) if lease_generation is None: raise RuntimeError("runtime resume result omitted lease_generation") except Exception as exc: await send_json( { "type": "error", "error": str(exc), "code": "runtime_resume_failed", } ) return None if not callable(attachment_send) or not callable(attachment_close): raise RuntimeError("session.resume requires transport attachment callbacks") def activation_payload_factory(token, generation: int) -> dict[str, object]: internal = { "type": "session.resumed", "session_id": session_id, "incarnation": incarnation, "attachment_generation": generation, "resume_token": token.plaintext, } return realtime_protocol.encode_outbound_event(internal)[0] try: resumed = await self._attachment_registry.resume( session_id, incarnation=incarnation, resume_token=resume_token, last_received_server_event_seq=last_received, send=attachment_send, close=attachment_close, activation_payload_factory=activation_payload_factory, ) except Exception as exc: # The engine-side CAS already advanced even if the transport # vanished before it received the rotated token. Keep that # generation so the registry's one-shot recovery token can retry. self._lease_generations[session_id] = lease_generation await send_json( { "type": "error", "error": str(exc), "code": "session_resume_conflict", } ) return None self._lease_generations[session_id] = lease_generation replaced = resumed.replaced_attachment if replaced is not None: replaced_payload = realtime_protocol.encode_outbound_event( { "type": "session.replaced", "session_id": session_id, "attachment_generation": replaced.generation, } )[0] with suppress(Exception): await replaced.send(replaced_payload) with suppress(Exception): await replaced.close("session_replaced") return _DuplexSessionHandshake( session=session, resumed=True, attachment_generation=resumed.attachment_generation, ) @classmethod def _runtime_lease_generation(cls, result: object) -> int | None: if isinstance(result, dict): generation = result.get("lease_generation") if isinstance(generation, int): return generation for key in ("stage_results", "result"): generation = cls._runtime_lease_generation(result.get(key)) if generation is not None: return generation return None if isinstance(result, list | tuple): for item in result: generation = cls._runtime_lease_generation(item) if generation is not None: return generation return None def _uses_serving_runtime_adapter(self, config: DuplexSessionConfig) -> bool: adapter = self._serving_runtime_adapter return adapter is not None and adapter.is_enabled(config) def _runtime_session_update_error( self, session: DuplexSession, payload: dict[str, object], ) -> dict[str, object] | None: if not self._uses_native_input_append(session): return None try: self._require_serving_runtime_adapter().validate_client_extra_body(payload.get("extra_body")) except ServingRuntimeConfigError as exc: return { "type": "error", "session_id": session.session_id, "code": exc.code, "error": str(exc), } return None def _runtime_config_for_session_update( self, session: DuplexSession, candidate_config: DuplexSessionConfig, ) -> dict[str, object]: if not self._uses_native_input_append(session): return dict(session.runtime_config) return self._require_serving_runtime_adapter().runtime_config_for_update( candidate_config, dict(session.runtime_config), ) def _resolve_server_vad_defaults( self, session: DuplexSession, config: DuplexSessionConfig, payload: dict[str, object], ) -> None: """Resolve omitted options without overriding explicit client policies.""" if config.server_vad is None: return if config.server_vad.interrupt_response is None: config.server_vad = replace(config.server_vad, interrupt_response=self._uses_native_input_append(session)) extra = payload.get("extra_body") overlap_policy = extra.get("overlap_policy") if isinstance(extra, dict) else None if not isinstance(overlap_policy, str): overlap_policy = payload.get("overlap_policy") config.overlap_policy = ( overlap_policy if isinstance(overlap_policy, str) else "barge_in_on_speech" if config.server_vad.interrupt_response else "listen_only" ) def _runtime_session_candidate_update_error( self, session: DuplexSession, candidate_config: DuplexSessionConfig, *, realtime_protocol: NativeRealtimeSessionProtocol | None = None, ) -> dict[str, object] | None: if candidate_config.server_vad is not None: desired_policy = "barge_in_on_speech" if candidate_config.server_vad.interrupt_response else "listen_only" if candidate_config.overlap_policy != desired_policy: return { "type": "error", "code": "unsupported_turn_detection", "error": f"overlap_policy conflicts with turn_detection; expected {desired_policy!r}", "param": "overlap_policy", } if not self._uses_native_input_append(session): if ( candidate_config.server_vad is not None and candidate_config.server_vad.min_speech_duration_ms is not None ): return { "type": "error", "code": "unsupported_turn_detection", "error": "server_vad.min_speech_duration_ms is only supported by model-native duplex runtimes", } requests_interruption = ( candidate_config.server_vad is not None and candidate_config.server_vad.interrupt_response ) or candidate_config.overlap_policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value if requests_interruption: return { "type": "error", "code": "server_vad_requires_native_duplex", "error": ( "Realtime server_vad interruption requires a model-native duplex runtime; " "set interrupt_response=false for turn-based models" ), } if candidate_config.server_vad is not None and realtime_protocol is not None: session_payload = candidate_config.extra_body.get("realtime_session_payload") audio_error = realtime_protocol.validate_server_vad_input_audio( session_payload if isinstance(session_payload, dict) else {} ) if audio_error is not None: return { "type": "error", "code": "unsupported_turn_detection", "error": audio_error, "param": "audio.input.format", } return None if candidate_config.server_vad is not None: if not candidate_config.server_vad.interrupt_response: return { "type": "error", "session_id": session.session_id, "code": "unsupported_turn_detection", "error": "Model-native duplex server_vad requires interrupt_response=true", } if candidate_config.server_vad.threshold <= SILERO_VAD_MIN_THRESHOLD: return { "type": "error", "session_id": session.session_id, "code": "unsupported_turn_detection", "error": ( f"Model-native duplex server_vad.threshold must be greater than {SILERO_VAD_MIN_THRESHOLD}" ), } if not candidate_config.server_vad.create_response: return { "type": "error", "session_id": session.session_id, "code": "unsupported_turn_detection", "error": "Model-native duplex server_vad requires create_response=true", } if ( candidate_config.instructions != session.config.instructions and self._runtime_session_state(session).native_context_locked ): return { "type": "error", "session_id": session.session_id, "code": "instructions_update_unsupported", "error": "session.update cannot change instructions after the native duplex context is initialized", } if native_duplex_opt_in(session.config.extra_body) is not True: return None if not self._config_requests_audio_output(candidate_config): return None if "ref_audio_data" in session.runtime_config: return None return { "type": "error", "session_id": session.session_id, "code": "ref_audio_required", "error": "MiniCPM-o native duplex audio output requires ref_audio", } @staticmethod def _uses_native_input_append(session: DuplexSession) -> bool: return ( session.capabilities.implementation_level == "model_native_duplex" and session.capabilities.supports_input_append and native_duplex_opt_in(session.config.extra_body) is not False ) @staticmethod def _config_requests_audio_output(config: DuplexSessionConfig) -> bool: return any(str(modality).lower() == "audio" for modality in config.modalities) @staticmethod def _native_stage0_request_id(session: DuplexSession, epoch: int) -> str: return duplex_resource_request_id( DuplexFence( session.session_id, epoch=epoch, incarnation=session.incarnation, ), "stage0", ) @staticmethod def _session_auto_responds(session: DuplexSession) -> bool: """Full-duplex / model-driven mode. When set, the server runs per-chunk speak-generation continuously (like the official MiniCPM-o ``duplex_generate`` loop) instead of waiting for an explicit ``response.create``: each ~chunk_period of appended audio is emitted and fed to the stage0 stream so the model itself decides to speak or listen. Signaled by the client via ``extra_body.auto_response`` (or ``extra_body.full_duplex``). """ extra = getattr(session.config, "extra_body", None) if not isinstance(extra, dict): return False return extra.get("auto_response") is True or extra.get("full_duplex") is True async def _receive_text( self, websocket: WebSocket, timeout_s: float, *, realtime_protocol: NativeRealtimeSessionProtocol | None = None, ) -> str | None: try: if realtime_protocol is not None: return await asyncio.wait_for( realtime_protocol.receive_internal_event_text(websocket), timeout=max(0.1, timeout_s), ) return await asyncio.wait_for(websocket.receive_text(), timeout=max(0.1, timeout_s)) except asyncio.TimeoutError: return None @staticmethod def _input_committed_payload( session: DuplexSession, committed: DuplexCommittedInput, *, item_id: object | None = None, ) -> dict[str, object]: payload: dict[str, object] = { "type": "input.committed", "session_id": session.session_id, "turn_id": committed.turn_id, "epoch": committed.epoch, "history_len": len(session.history), "message": committed.message, } if isinstance(item_id, str) and item_id: payload["item_id"] = item_id return payload @staticmethod def _commit_native_audio_input( session: DuplexSession, *, item_id: object | None = None, transcript: object | None = None, turn_id: int | None = None, ) -> DuplexCommittedInput: clean_transcript = transcript.strip() if isinstance(transcript, str) else None committed = session.commit_native_audio_input( transcript=clean_transcript or None, turn_id=turn_id, ) if isinstance(item_id, str) and item_id: session.register_history_item(item_id, committed.message) return committed @staticmethod def _native_audio_committed_payload( session: DuplexSession, *, committed: DuplexCommittedInput | None = None, item_id: object | None = None, transcript: object | None = None, ) -> dict[str, object]: message = committed.message if committed is not None else None if not isinstance(message, dict): input_audio_part: dict[str, object] = { "type": "audio_url", "audio_url": {"url": "native-duplex:input-audio"}, } if isinstance(transcript, str) and transcript: input_audio_part["transcript"] = transcript message = { "role": "user", "content": [input_audio_part], } payload: dict[str, object] = { "type": "input.committed", "session_id": session.session_id, "turn_id": committed.turn_id if committed is not None else session.turn_id, "epoch": committed.epoch if committed is not None else session.epoch, "history_len": len(session.history), "native_audio": True, "message": message, } if isinstance(transcript, str) and transcript: payload["transcript"] = transcript if isinstance(item_id, str) and item_id: payload["item_id"] = item_id return payload def _apply_session_update( self, session: DuplexSession, payload: dict[str, object], ) -> dict[str, object] | None: model = payload.get("model") audio_config = payload.get("audio") audio_input = audio_config.get("input") if isinstance(audio_config, dict) else None audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None voice = payload.get("voice") if not isinstance(voice, str) and isinstance(audio_output, dict): voice = audio_output.get("voice") if isinstance(model, str) and session.config.model is not None and model != session.config.model: return { "type": "error", "session_id": session.session_id, "code": "model_update_unsupported", "error": "session.update cannot change model for an open realtime duplex session", } if isinstance(model, str) and session.config.model is None: session.config.model = model if isinstance(voice, str) and (session.playback.generated_ms > 0 or session.playback.sent_ms > 0): return { "type": "error", "session_id": session.session_id, "code": "voice_update_after_audio_unsupported", "error": "session.update cannot change voice after audio output has started", } if isinstance(payload.get("ref_audio"), str): return { "type": "error", "session_id": session.session_id, "code": "ref_audio_update_unsupported", "error": "session.update cannot change ref_audio after the native duplex runtime is open", } extra_body_payload = payload.get("extra_body") if isinstance(extra_body_payload, dict): # Fold the deprecated model-prefixed alias into the canonical # native_duplex key before comparing or merging. extra_body_payload = normalize_native_duplex_key(dict(extra_body_payload)) payload = {**payload, "extra_body": extra_body_payload} if isinstance(extra_body_payload, dict) and NATIVE_DUPLEX_KEY in extra_body_payload: requested_native_duplex = extra_body_payload[NATIVE_DUPLEX_KEY] if not isinstance(requested_native_duplex, bool): return { "type": "error", "session_id": session.session_id, "code": "invalid_request_error", "error": "extra_body.native_duplex must be a boolean", } current_native_duplex = native_duplex_opt_in(session.config.extra_body) if current_native_duplex is not None and extra_body_payload[NATIVE_DUPLEX_KEY] != current_native_duplex: return { "type": "error", "session_id": session.session_id, "code": "native_duplex_mode_update_unsupported", "error": "session.update cannot change native_duplex after the session is created", } if isinstance(payload.get("instructions"), str): session.config.instructions = str(payload["instructions"]) elif "instructions" in payload and payload.get("instructions") is None: session.config.instructions = None if isinstance(voice, str): session.config.voice = str(voice) elif "voice" in payload and payload.get("voice") is None: session.config.voice = None response_format = payload.get("output_audio_format") or payload.get("response_format") if response_format is None and isinstance(audio_config, dict): if isinstance(audio_output, dict): response_format = audio_output.get("format") response_format, _ = NativeRealtimeSessionProtocol._parse_realtime_audio_format(response_format) if isinstance(response_format, str) and response_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: session.config.response_format = NativeRealtimeSessionProtocol._duplex_response_format(response_format) if isinstance(payload.get("temperature"), int | float): session.config.temperature = float(payload["temperature"]) speed = payload.get("speed") if not isinstance(speed, int | float) and isinstance(audio_output, dict): speed = audio_output.get("speed") if isinstance(speed, int | float): session.config.speed = float(speed) max_tokens = ( payload.get("max_response_output_tokens") if "max_response_output_tokens" in payload else payload.get("max_output_tokens") if "max_output_tokens" in payload else payload.get("max_tokens") ) if "max_response_output_tokens" in payload or "max_output_tokens" in payload or "max_tokens" in payload: session.config.max_tokens = NativeRealtimeSessionProtocol.realtime_max_output_tokens(max_tokens) if isinstance(payload.get("overlap_policy"), str): session.config.overlap_policy = DuplexSessionConfig._normalize_overlap_policy( str(payload["overlap_policy"]) ) if isinstance(payload.get("overlap_short_ack_ms"), int | float): session.config.overlap_short_ack_ms = max(0, int(payload["overlap_short_ack_ms"])) if isinstance(payload.get("overlap_barge_in_ms"), int | float): session.config.overlap_barge_in_ms = max(0, int(payload["overlap_barge_in_ms"])) if isinstance(payload.get("overlap_silence_rms"), int | float): session.config.overlap_silence_rms = max(0.0, float(payload["overlap_silence_rms"])) if isinstance(payload.get("playback_commit_policy"), str): session.config.playback_commit_policy = DuplexSessionConfig._normalize_playback_commit_policy( str(payload["playback_commit_policy"]) ) modalities = payload.get("modalities") or payload.get("output_modalities") if isinstance(modalities, list) and all(isinstance(item, str) for item in modalities): session.config.modalities = list(modalities) if isinstance(payload.get("extra_body"), dict): session.config.extra_body.update(payload["extra_body"]) extra = payload["extra_body"] if isinstance(extra.get("overlap_policy"), str): session.config.overlap_policy = DuplexSessionConfig._normalize_overlap_policy( str(extra["overlap_policy"]) ) if isinstance(extra.get("playback_commit_policy"), str): session.config.playback_commit_policy = DuplexSessionConfig._normalize_playback_commit_policy( str(extra["playback_commit_policy"]) ) if isinstance(payload.get("tools"), list): session.config.extra_body["realtime_tools"] = payload["tools"] elif "tools" in payload and payload.get("tools") is None: session.config.extra_body.pop("realtime_tools", None) if isinstance(payload.get("tool_choice"), str | dict): session.config.extra_body["realtime_tool_choice"] = payload["tool_choice"] elif "tool_choice" in payload and payload.get("tool_choice") is None: session.config.extra_body.pop("realtime_tool_choice", None) if isinstance(payload.get("metadata"), dict): session.config.extra_body["realtime_metadata"] = dict(payload["metadata"]) elif "metadata" in payload and payload.get("metadata") is None: session.config.extra_body.pop("realtime_metadata", None) if isinstance(payload.get("include"), list): session.config.extra_body["realtime_include"] = list(payload["include"]) elif "include" in payload and payload.get("include") is None: session.config.extra_body.pop("realtime_include", None) if isinstance(payload.get("prompt"), dict): session.config.extra_body["realtime_prompt"] = dict(payload["prompt"]) elif "prompt" in payload and payload.get("prompt") is None: session.config.extra_body.pop("realtime_prompt", None) input_audio_transcription = NativeRealtimeSessionProtocol._input_audio_transcription_config(payload) if isinstance(input_audio_transcription, dict): session.config.extra_body["realtime_input_audio_transcription"] = dict(input_audio_transcription) elif "input_audio_transcription" in payload and payload.get("input_audio_transcription") is None: session.config.extra_body.pop("realtime_input_audio_transcription", None) if isinstance(payload.get("input_audio_noise_reduction"), dict): session.config.extra_body["realtime_input_audio_noise_reduction"] = dict( payload["input_audio_noise_reduction"] ) elif "input_audio_noise_reduction" in payload and payload.get("input_audio_noise_reduction") is None: session.config.extra_body.pop("realtime_input_audio_noise_reduction", None) if isinstance(audio_input, dict) and isinstance(audio_input.get("noise_reduction"), dict): session.config.extra_body["realtime_input_audio_noise_reduction"] = dict(audio_input["noise_reduction"]) elif isinstance(audio_input, dict) and audio_input.get("noise_reduction") is None: session.config.extra_body.pop("realtime_input_audio_noise_reduction", None) if isinstance(payload.get("audio"), dict): session.config.extra_body["realtime_audio"] = dict(payload["audio"]) elif "audio" in payload and payload.get("audio") is None: session.config.extra_body.pop("realtime_audio", None) if isinstance(payload.get("tracing"), str | dict): session.config.extra_body["realtime_tracing"] = payload["tracing"] elif "tracing" in payload and payload.get("tracing") is None: session.config.extra_body.pop("realtime_tracing", None) session.config.extra_body["realtime_session_payload"] = ( NativeRealtimeSessionProtocol._json_safe_realtime_payload(payload) ) try: turn_detection_configured, server_vad = parse_session_turn_detection(payload) if turn_detection_configured: vad_config = replace(session.config, server_vad=server_vad) self._resolve_server_vad_defaults(session, vad_config, payload) server_vad = vad_config.server_vad if ( not self._uses_native_input_append(session) and session.turn_detection_config_locked and (not session.config.turn_detection_configured or server_vad != session.config.server_vad) ): return { "type": "error", "session_id": session.session_id, "code": "unsupported_turn_detection", "error": "turn_detection cannot be changed after the first audio append", } session.config.turn_detection_configured = True session.config.server_vad = server_vad session.config.overlap_policy = vad_config.overlap_policy except ValueError as exc: return { "type": "error", "session_id": session.session_id, "code": "invalid_request_error", "error": str(exc), } return None async def _configure_server_vad(self, session: DuplexSession) -> None: pipeline = await self._prepare_server_vad_pipeline(session, session.config) self._install_server_vad_pipeline(session, pipeline) def _install_server_vad_pipeline( self, session: DuplexSession, pipeline: ServerVADPipeline | None, ) -> None: existing = self._server_vad_pipelines.pop(session.session_id, None) if existing is not None: existing.reset() self._realtime_vad_metrics.session_finished() if pipeline is None: return self._server_vad_pipelines[session.session_id] = pipeline self._realtime_vad_metrics.session_started() async def _prepare_server_vad_pipeline( self, session: DuplexSession, config: DuplexSessionConfig, ) -> ServerVADPipeline | None: if config.server_vad is None: return None backend = await asyncio.to_thread(self._server_vad_backend_provider.get) pipeline_config = config.server_vad if self._uses_native_input_append(session) and pipeline_config.min_speech_duration_ms is None: pipeline_config = replace( pipeline_config, min_speech_duration_ms=SILERO_VAD_DEFAULT_MIN_SPEECH_DURATION_MS, ) return ServerVADPipeline(backend, pipeline_config) def _apply_response_create_options( self, session: DuplexSession, payload: dict[str, object], ) -> str | None: """Reserve options that apply only to the next response lifecycle.""" audio_config = payload.get("audio") audio_output = audio_config.get("output") if isinstance(audio_config, dict) else None if session.capabilities.implementation_level == "model_native_duplex": nested_voice = audio_output.get("voice") if isinstance(audio_output, dict) else None unsupported = ( payload.get("instructions") is not None or payload.get("voice") is not None or nested_voice is not None or payload.get("temperature") is not None or any( payload.get(field_name) is not None for field_name in ("max_response_output_tokens", "max_output_tokens", "max_tokens") ) or payload.get("tools") is not None or payload.get("tool_choice") is not None ) if unsupported: return "unsupported_native_response_options" instructions = str(payload["instructions"]) if isinstance(payload.get("instructions"), str) else None voice = payload.get("voice") if not isinstance(voice, str) and isinstance(audio_output, dict): voice = audio_output.get("voice") voice = str(voice) if isinstance(voice, str) else None response_format = payload.get("output_audio_format") or payload.get("response_format") if response_format is None and isinstance(audio_config, dict): if isinstance(audio_output, dict): response_format = audio_output.get("format") response_format, _ = NativeRealtimeSessionProtocol._parse_realtime_audio_format(response_format) if isinstance(response_format, str) and response_format.lower() in REALTIME_OUTPUT_AUDIO_FORMATS: response_format = NativeRealtimeSessionProtocol._duplex_response_format(response_format) else: response_format = None temperature = float(payload["temperature"]) if isinstance(payload.get("temperature"), int | float) else None speed = payload.get("speed") if not isinstance(speed, int | float) and isinstance(audio_output, dict): speed = audio_output.get("speed") speed = float(speed) if isinstance(speed, int | float) else None max_tokens = ( payload.get("max_response_output_tokens") if "max_response_output_tokens" in payload else payload.get("max_output_tokens") if "max_output_tokens" in payload else payload.get("max_tokens") ) if "max_response_output_tokens" in payload or "max_output_tokens" in payload or "max_tokens" in payload: max_tokens = NativeRealtimeSessionProtocol.realtime_max_output_tokens(max_tokens) else: max_tokens = None modalities = payload.get("modalities") or payload.get("output_modalities") if isinstance(modalities, list) and all(isinstance(item, str) for item in modalities): modalities = tuple(modalities) else: modalities = None response_extra: dict[str, object] = {} conversation = payload.get("conversation") if isinstance(conversation, str): response_extra["realtime_response_conversation"] = conversation metadata = payload.get("metadata") if isinstance(metadata, dict): response_extra["realtime_response_metadata"] = dict(metadata) prompt = payload.get("prompt") if isinstance(prompt, dict): response_extra["realtime_response_prompt"] = dict(prompt) if isinstance(payload.get("tools"), list): response_extra["realtime_response_tools"] = payload["tools"] if isinstance(payload.get("tool_choice"), str | dict): response_extra["realtime_response_tool_choice"] = payload["tool_choice"] extra_body = payload.get("extra_body") if isinstance(extra_body, dict): if session.capabilities.implementation_level == "model_native_duplex": response_extra.update( (key, value) for key, value in extra_body.items() if key not in self._require_serving_runtime_adapter().private_runtime_config_keys ) else: response_extra.update(extra_body) try: session.reserve_response_options( ResponseCreateOptions( instructions=instructions, voice=voice, response_format=response_format, temperature=temperature, max_tokens=max_tokens, speed=speed, modalities=modalities, extra_body=response_extra, ) ) except RuntimeError: return "response_already_active" return None @staticmethod def _realtime_item_to_history_message(item: object) -> dict[str, object] | None: if not isinstance(item, dict): return None role = item.get("role") if role not in {"system", "user", "assistant"}: return None content = item.get("content") if isinstance(content, str): text = content.strip() return {"role": role, "content": text} if text else None if not isinstance(content, list): return None text_chunks: list[str] = [] audio_chunks: list[dict[str, object]] = [] for part in content: if not isinstance(part, dict): continue part_type = part.get("type") if part_type in {"input_text", "text", "output_text"} and isinstance(part.get("text"), str): text_chunks.append(str(part["text"])) elif part_type in {"input_audio", "audio"}: audio = part.get("audio") or part.get("data") fmt = part.get("format") if isinstance(part.get("format"), str) else "wav" if isinstance(audio, str) and audio: audio_chunks.append( { "type": "audio_url", "audio_url": { "url": f"data:audio/{fmt};base64,{audio}", }, } ) elif part_type in {"audio_transcript", "transcript"} and isinstance(part.get("text"), str): text_chunks.append(str(part["text"])) text = "".join(text_chunks).strip() if audio_chunks: content_items: list[dict[str, object]] = [] if text: content_items.append({"type": "text", "text": text}) content_items.extend(audio_chunks) return {"role": role, "content": content_items} if text: return {"role": role, "content": text} return None async def _handle_playback_ack(self, session: DuplexSession, event: dict[str, object], send_json) -> None: played_ms = event.get("played_ms", event.get("audio_ms", 0)) committed_ms = event.get("committed_ms") if not isinstance(played_ms, int | float): await send_json({"type": "error", "error": "playback.ack requires played_ms", "code": "bad_event"}) return committed_cursor = int(committed_ms) if isinstance(committed_ms, int | float) else int(played_ms) item_id = event.get("item_id") response_id = event.get("response_id") response_id = response_id if isinstance(response_id, str) and response_id else None if not isinstance(item_id, str) or not item_id: item_id = f"item_{response_id}" if response_id is not None else None elif response_id is None and item_id.startswith("item_"): response_id = item_id.removeprefix("item_") if response_id is None and item_id is None and len(session.pending_history_item_ids) == 1: item_id = next(iter(session.pending_history_item_ids)) if item_id.startswith("item_"): response_id = item_id.removeprefix("item_") if response_id is None and item_id is None and session.active_response_id is not None: response_id = session.active_response_id item_id = f"item_{response_id}" if response_id is not None: expected_item_id = f"item_{response_id}" if item_id is None: item_id = expected_item_id elif item_id != expected_item_id: await send_json( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": "playback_item_mismatch", "error": "playback.ack item_id must match item_.", } ) return if not session.has_assistant_response_item(response_id, item_id): await send_json( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": "playback_item_not_found", "error": f"No assistant response item is registered for {response_id}.", } ) return if session.playback_ack_is_too_late(response_id, item_id): await send_json( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": "playback_ack_too_late", "error": "playback.ack arrived after a later user input was committed.", } ) return # Reserve the response's current history position before any later # input commit can append a user item. A 0 ms ACK is intentional: # the response may be active but have no audio delta yet. session.reserve_history_item(item_id) elif item_id is not None: await send_json( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": "playback_item_not_found", "error": "playback.ack requires a response-owned assistant item.", } ) return hard_truncate = event.get("truncate") is True if hard_truncate: playback = session.acknowledge_playback( int(played_ms), committed_cursor, response_id=response_id, ) playback = session.truncate_playback_commit( committed_cursor, response_id=response_id, ) else: playback = session.acknowledge_playback( int(played_ms), committed_cursor, response_id=response_id, ) committed_history = False if isinstance(item_id, str) and item_id: expected_item_id = f"item_{response_id}" if response_id is not None else None if ( expected_item_id == item_id and item_id not in session.history_item_ids and item_id not in session.pending_history_item_ids ): # ACK_ONLY responses are normally registered as pending when # the response ends. Recover the response-local item if that # registration was lost across the response/ack boundary. session.register_history_item(item_id, None) committed_history = session.truncate_history_item( item_id, audio_end_ms=committed_cursor, playback=playback, hard=hard_truncate, ) elif session.pending_history_item_ids: # A plain playback ack has no OpenAI item id. Commit the only # uncommitted assistant candidate if the session has an unambiguous # pending response; otherwise wait for conversation.item.truncate. pending_ids = list(session.pending_history_item_ids) if len(pending_ids) == 1: item_id = pending_ids[0] committed_history = session.truncate_history_item( item_id, audio_end_ms=committed_cursor, playback=playback, hard=hard_truncate, ) elif session.active_response_id is not None: item_id = f"item_{session.active_response_id}" committed_history = session.truncate_history_item( item_id, audio_end_ms=committed_cursor, playback=playback, hard=hard_truncate, ) elif session.last_assistant_full_message is not None: if item_id is None and session.history_item_ids: assistant_item_ids = [ known_item_id for known_item_id, message in session.history_item_ids.items() if message.get("role") == "assistant" ] if len(assistant_item_ids) == 1: item_id = assistant_item_ids[0] if isinstance(item_id, str) and item_id: committed_history = session.truncate_history_item( item_id, audio_end_ms=committed_cursor, playback=playback, hard=hard_truncate, ) await send_json( { "type": "playback.acknowledged", "session_id": session.session_id, "epoch": session.epoch, "item_id": item_id, "played_ms": int(played_ms), "committed_ms": committed_cursor, "truncate": event.get("truncate") is True, "playback": playback.as_dict(), "history_committed": committed_history, } ) if committed_history and committed_cursor >= max(playback.sent_ms, playback.generated_ms): session.release_response_playback(response_id) session.release_response_history_snapshot(response_id) async def _cancel_active_response( self, session: DuplexSession, active_task: asyncio.Task[None] | None, send_json, *, reason: str, notify: bool = True, ) -> bool: has_running_task = active_task is not None and not active_task.done() if not has_running_task and session.active_request_id is None and session.active_response_id is None: return False old_epoch = session.epoch old_request_id = session.active_request_id old_response_id = session.active_response_id committed_ms = session.playback.committed_ms committed_message = session.end_response( commit_text=self._should_commit_response_to_history(session, old_response_id), playback_commit_policy=DuplexPlaybackCommitPolicy.ACK_ONLY.value, ) if old_response_id is not None: item_id = f"item_{old_response_id}" if committed_message is not None: session.register_history_item(item_id, committed_message) elif committed_ms > 0 and not session.playback_ack_is_too_late(old_response_id, item_id): session.truncate_history_item(item_id, audio_end_ms=committed_ms) new_epoch, old_playback = self._advance_barge_in_epoch(session) if old_request_id is not None: # Epoch-scoped request ids prevent state reuse, but explicitly # release projector/parser cursors so cancelled epochs do not # accumulate until the whole session closes. if self._serving_runtime_adapter is not None: self._serving_runtime_adapter.data_plane.close_stream(old_request_id) await self._abort_request_background( session, old_request_id, send_json, notify=notify, ) if has_running_task and active_task is not None: active_task.cancel() try: await asyncio.wait_for(asyncio.gather(active_task, return_exceptions=True), timeout=0.25) except asyncio.TimeoutError: pass if notify: await send_json( { "type": "audio.cancelled", "session_id": session.session_id, "response_id": old_response_id, "reason": reason, "cancelled_epoch": old_epoch, "epoch": new_epoch, "committed_ms": committed_ms, "playback": old_playback, } ) return True async def _abort_request_background( self, session: DuplexSession, request_id: str, send_json, *, notify: bool, ) -> None: try: abort_internal = getattr(self._chat_service.engine_client, "_abort_internal_requests", None) if callable(abort_internal): result = abort_internal([request_id]) else: result = self._chat_service.engine_client.abort([request_id]) if inspect.isawaitable(result): await result except Exception as exc: logger.exception("Failed to abort duplex request %s: %s", request_id, exc) if notify and session.state != DuplexSessionState.CLOSED: await self._send_runtime_error(send_json, "runtime_abort_failed", exc, session=session) async def _cancel_pending_input(self, session: DuplexSession, send_json, *, reason: str) -> None: cancelled = session.cancel_pending_input() self._advance_barge_in_epoch(session) await send_json( { "type": "input.cancelled", "session_id": session.session_id, "reason": reason, "epoch": session.epoch, "cancelled": cancelled, } )