# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project from __future__ import annotations import asyncio import binascii import json import time import uuid from collections.abc import Callable from contextlib import suppress from copy import deepcopy import numpy as np import pybase64 as base64 from fastapi import WebSocket, WebSocketDisconnect from vllm.logger import init_logger from vllm_omni.engine.duplex.lease import DuplexLeaseActivity from vllm_omni.engine.duplex.messages import DuplexFence from vllm_omni.entrypoints.duplex.audio import ( convert_input_audio_with_rate, validate_input_sample_rate_hz, ) from vllm_omni.entrypoints.duplex.commit_policy import ( CommitAction, CommitSnapshot, decide_commit_action, ) from vllm_omni.entrypoints.duplex.protocol import ( DuplexOverlapPolicy, DuplexPlaybackCommitPolicy, DuplexSession, DuplexSessionState, DuplexTurnEventType, ) from vllm_omni.entrypoints.duplex.realtime_session import ( NativeRealtimeSessionProtocol, ) from vllm_omni.entrypoints.duplex.runtime_adapter import ( PcmAppendReservation, ServingRuntimeConfigError, ServingRuntimeSessionState, payload_turn_id, ) from vllm_omni.entrypoints.duplex.server_vad import ServerVADPipeline from vllm_omni.entrypoints.duplex.session_attachment import ( DuplexJournalOverflowError, ) from vllm_omni.entrypoints.duplex.websocket import ( DuplexWebSocketActor, is_input_event, normalize_duplex_input_event, ) logger = init_logger(__name__) _MAX_EVENT_BYTES = 15 * 1024 * 1024 class DuplexSessionRunnerMixin: """Run one ordered WebSocket mailbox for a duplex session.""" async def handle_session( self, websocket: WebSocket, *, realtime_protocol: NativeRealtimeSessionProtocol | None = None, ) -> None: await websocket.accept() session: DuplexSession | None = None actor = DuplexWebSocketActor( websocket, current_epoch=lambda: session.epoch if session is not None else None, session_closed=lambda: session is not None and session.state == DuplexSessionState.CLOSED, outbound_protocol=realtime_protocol, ) runtime_opened = False runtime_closed = False attachment_ready = False attachment_generation: int | None = None resume_credential_delivered = False transport_detached = False pending_turn_reservations = 0 async def attachment_send(payload: dict[str, object]) -> None: nonlocal resume_credential_delivered try: await websocket.send_json(payload) except RuntimeError as exc: message = str(exc) if "after sending 'websocket.close'" in message or "response already completed" in message: raise WebSocketDisconnect(code=1006) from exc raise if payload.get("type") == "session.created" and isinstance(payload.get("resume_token"), str): resume_credential_delivered = True async def attachment_close(reason: str) -> None: close = getattr(websocket, "close", None) if callable(close): await close(code=1000, reason=reason) async def send_attachment_payload( payload: dict[str, object], *, journal: bool, on_accepted: Callable[[], None] | None = None ) -> None: assert session is not None session_id = session.session_id should_journal = journal and session_id not in self._resync_required_sessions try: await self._attachment_registry.send_event( session_id, payload, journal=should_journal, on_accepted=on_accepted, ) except DuplexJournalOverflowError: first_overflow = session_id not in self._resync_required_sessions self._resync_required_sessions.add(session_id) if first_overflow: resync_payload: dict[str, object] = { "type": "session.resync_required", "session_id": session_id, "reason": "journal_overflow", } if realtime_protocol is not None: resync_payload = realtime_protocol.encode_outbound_event(resync_payload)[0] await self._attachment_registry.send_event( session_id, resync_payload, journal=False, ) await self._attachment_registry.send_event( session_id, payload, journal=False, on_accepted=on_accepted, ) if realtime_protocol is not None: async def send_realtime_raw(payload: dict[str, object]) -> None: if attachment_ready and session is not None: await send_attachment_payload( payload, journal=payload.get("type") not in {"session.created", "session.resumed"}, ) return raw_payload = dict(payload) raw_payload["_realtime_raw"] = True await actor.send_json(raw_payload) realtime_protocol.bind_sender(send_realtime_raw) native: ServingRuntimeSessionState = self._create_runtime_session_state() def begin_close(reason: str) -> None: actor.closing = True actor.close_reason = reason if session is not None: session.mark_closing() event_emit_lock = asyncio.Lock() async def send_outbound(payload: dict[str, object], *, on_accepted: Callable[[], None] | None = None) -> None: if not attachment_ready or session is None: await actor.send_json(dict(payload)) if on_accepted is not None: on_accepted() return if realtime_protocol is None: await send_attachment_payload( payload, journal=payload.get("type") not in {"session.created", "session.resumed"}, on_accepted=on_accepted, ) return for projected in realtime_protocol.encode_outbound_event(payload): await send_attachment_payload( projected, journal=projected.get("type") not in {"session.created", "session.resumed"}, # Item/part announcements may precede the audio. Only the # actual audio event establishes acceptance of this chunk. on_accepted=on_accepted if projected.get("type") == "response.audio.delta" else None, ) async def emit_event(payload: dict[str, object], *, on_accepted: Callable[[], None] | None = None) -> None: deferred_precreate_response = False async with event_emit_lock: accepted, deferred_overlap_payload = await self._apply_outbound_session_event( payload, session=session, actor=actor, native=native, realtime_protocol=realtime_protocol, ) if not accepted: return await send_outbound(payload, on_accepted=on_accepted) if deferred_overlap_payload is not None: deferred_precreate_response = native.deferred_precreate_response native.deferred_precreate_response = False if deferred_overlap_payload is not None and session is not None and not actor.closing: await start_native_append( deferred_overlap_payload, final=True, precreate_response=deferred_precreate_response, operation_id=native.committed_audio_operation_id, retained_committed_payload=( deferred_overlap_payload if native.committed_audio_payload is deferred_overlap_payload else None ), ) writer_task = asyncio.create_task(actor.writer_loop(), name="duplex-session-writer") reader_task: asyncio.Task[None] | None = None async def read_event_loop() -> None: nonlocal pending_turn_reservations assert session is not None try: while not actor.closing: raw = await self._receive_text( websocket, session.config.idle_timeout_s, realtime_protocol=realtime_protocol, ) if raw is None: await actor.enqueue_event({"type": "__timeout__"}) return if len(raw.encode("utf-8")) > _MAX_EVENT_BYTES: await emit_event( {"type": "error", "error": "Duplex event too large", "code": "event_too_large"} ) continue try: event = json.loads(raw) except json.JSONDecodeError: await emit_event({"type": "error", "error": "Invalid JSON event", "code": "invalid_json"}) continue if not isinstance(event, dict): await emit_event( { "type": "error", "error": "Duplex event must be a JSON object", "code": "bad_event", } ) continue event = normalize_duplex_input_event(event) event_type = event.get("type") if not isinstance(event_type, str): await emit_event( {"type": "error", "error": "Duplex event missing string type", "code": "bad_event"} ) continue if event_type in {"input.commit", "input_audio_buffer.commit"}: if not session.reserve_pending_turn( limit=self._duplex_session_config.max_pending_turns_per_session ): await emit_event( { "type": "error", "error": "Duplex session has too many pending input turns", "code": "input_backpressure", } ) continue event["_duplex_pending_turn_reserved"] = True pending_turn_reservations += 1 if is_input_event(event_type) and native_response_in_progress(): event["_duplex_overlap_candidate"] = True await actor.enqueue_event(event) except WebSocketDisconnect: await actor.enqueue_event({"type": "__disconnect__"}) except Exception as exc: await emit_event({"type": "error", "error": str(exc), "code": "realtime_input_failed"}) await actor.enqueue_event({"type": "__disconnect__"}) async def next_actor_event() -> dict[str, object]: nonlocal pending_turn_reservations, transport_detached event = await actor.next_event() if event.pop("_duplex_pending_turn_reserved", False) and session is not None: session.release_pending_turn() pending_turn_reservations = max(0, pending_turn_reservations - 1) if ( attachment_ready and attachment_generation is not None and session is not None and not await self._attachment_registry.is_current_attachment( session.session_id, attachment_generation, ) ): transport_detached = True return {"type": "__replaced_attachment__"} return event def native_response_in_progress() -> bool: if session is None: return False if session.active_response_id is not None: return True if ( session.config.playback_commit_policy == DuplexPlaybackCommitPolicy.ACK_ONLY.value and session.playback.sent_ms > session.playback.committed_ms ): return True if actor.active_response_task is not None and not actor.active_response_task.done(): return True if actor.has_response_bound_append_tasks(): return True return False def start_chat_response() -> None: assert session is not None async def run_and_notify() -> None: task = asyncio.current_task() try: await self._run_response(session, emit_event) finally: await actor.enqueue_event( { "type": "__chat_response_finished__", "_response_task": task, } ) actor.active_response_task = asyncio.create_task(run_and_notify()) def activate_waiting_server_vad_turn() -> None: assert session is not None pending_turn = session.activate_server_vad_turn() if pending_turn is not None and pending_turn.create_response: start_chat_response() def correlate_realtime_error( payload: dict[str, object], event_id: object, ) -> dict[str, object]: if isinstance(event_id, str) and event_id: payload["event_id"] = event_id return payload async def reject_and_clear_server_vad_input( pipeline: ServerVADPipeline, *, code: str, error: str, metric_reason: str, event_id: object, ) -> None: assert session is not None session.cancel_pending_input() pipeline.reset() self._realtime_vad_metrics.error(metric_reason) await emit_event( correlate_realtime_error( { "type": "error", "session_id": session.session_id, "code": code, "error": error, }, event_id, ) ) await emit_event( { "type": "input_audio_buffer.cleared", "session_id": session.session_id, } ) async def commit_server_vad_turn( *, item_id: str, create_response: bool, retain_input_bytes: int, event_id: object = None, ) -> None: assert session is not None # Retain ownership until the actor consumes the terminal event so # queued audio cannot overtake pending-turn activation. active_response = actor.active_response_task is not None if active_response and session.pending_server_vad_turn is not None: released = session.discard_uncommitted_server_vad_utterance() session.release_input_bytes(released) self._realtime_vad_metrics.error("pending_turn_backpressure") await emit_event( correlate_realtime_error( { "type": "error", "session_id": session.session_id, "code": "input_backpressure", "error": "Only one committed Server VAD turn may wait for the active response", }, event_id, ) ) return if not session.stage_server_vad_audio_for_commit(): return committed = session.commit_user_input( append_history=not active_response, retain_input_bytes=retain_input_bytes, ) if committed is None: return if active_response: session.stage_server_vad_turn( committed, create_response=create_response, item_id=item_id, ) else: session.register_history_item(item_id, committed.message) await emit_event( self._input_committed_payload( session, committed, item_id=item_id, ) ) if create_response and not active_response: start_chat_response() def clear_completed_pending_silence() -> None: task = native.pending_silence_task if task is not None and task.done(): native.pending_silence_task = None native.pending_silence_owner_id = None def mark_pending_silence_superseded() -> None: task = native.pending_silence_task if task is None: return if task.done(): native.pending_silence_task = None # Do not cancel the task or clear native_append_tail here. A # silence append may already have reached the Engine, and local # cancellation cannot retract that RPC. The sequencer preserves # wire order; before_append will skip silence that has not started. native.pending_silence_owner_id = None def real_native_input_waiting() -> bool: clear_completed_pending_silence() return ( native.audio_buffer.has_pending() or native.audio_buffer.has_reserved() or actor.has_queued_input_events() ) async def start_native_append( payload: object, *, final: bool, precreate_response: bool = False, pcm_reservation: PcmAppendReservation | None = None, operation_id: str | None = None, retained_committed_payload: dict[str, object] | None = None, silence_continuation: bool = False, before_append=None, ) -> asyncio.Task[bool] | None: if session is None: return if not silence_continuation: mark_pending_silence_superseded() append_epoch = session.epoch append_turn_id = payload_turn_id(payload) if append_turn_id is None: append_turn_id = session.turn_id request_id = self._native_stage0_request_id(session, append_epoch) if final or precreate_response: session.bind_request(request_id) if precreate_response: session.bind_response_turn(append_turn_id) if precreate_response and session.active_response_id is None: response_id = session.begin_response(turn_id=append_turn_id) await emit_event( self._response_created_payload( session, response_id, epoch=append_epoch, ) ) precreated_response_id = session.active_response_id if precreate_response else None def _discard_retained_committed_audio() -> None: if ( retained_committed_payload is not None and native.committed_audio_payload is retained_committed_payload ): session.release_input_bytes(native.clear_committed_audio()) async def _run() -> bool: nonlocal runtime_closed try: session.mark_model_turn_request_started(append_turn_id, time.monotonic()) append_ok, emitted_response = await self._append_runtime_input( session, payload, operation_id=(pcm_reservation.operation_id if pcm_reservation is not None else operation_id), final=final, send_json=emit_event, mode="append_audio_chunk", expected_epoch=append_epoch, ) if append_ok: native.native_context_locked = True if pcm_reservation is not None: pcm_reservation.commit() session.release_input_bytes(pcm_reservation.byte_count) if ( retained_committed_payload is not None and native.committed_audio_payload is retained_committed_payload ): session.release_input_bytes(native.clear_committed_audio()) else: if pcm_reservation is not None: pcm_reservation.rollback() _discard_retained_committed_audio() if ( not append_ok and precreated_response_id is not None and session.active_response_id == precreated_response_id ): session.end_response(commit_text=False) await emit_event( { "type": "response.done", "session_id": session.session_id, "response_id": precreated_response_id, "epoch": session.epoch, "committed": False, "status": "failed", "status_details": { "type": "failed", "reason": "runtime_append_failed", }, "playback": session.playback.as_dict(), } ) if not append_ok and session.state == DuplexSessionState.CLOSED: runtime_closed = True return False if not emitted_response and session.epoch == append_epoch: if session.active_request_id == self._native_stage0_request_id(session, append_epoch): session.clear_request(request_id) if final: await emit_event( self._turn_controller.signal(session, DuplexTurnEventType.USER_STARTED.value) ) return append_ok except asyncio.CancelledError: if pcm_reservation is not None: pcm_reservation.rollback() raise except Exception as exc: if pcm_reservation is not None: pcm_reservation.rollback() _discard_retained_committed_audio() logger.exception("Native duplex append task failed: %s", exc) await self._send_runtime_error(emit_event, "runtime_append_task_failed", exc, session=session) if session.state != DuplexSessionState.CLOSED: begin_close("runtime_append_task_failed") if await self._close_runtime_session( session, reason="runtime_append_task_failed", send_json=emit_event, ): runtime_closed = True session.close() await emit_event( { "type": "session.closed", "session_id": session.session_id, "reason": "runtime_append_task_failed", } ) return False async def _run_in_wire_order(predecessor: asyncio.Task[bool] | None) -> bool: if predecessor is not None: try: predecessor_ok = await predecessor except asyncio.CancelledError: current = asyncio.current_task() if current is not None and current.cancelling(): raise predecessor_ok = False except Exception: predecessor_ok = False if not predecessor_ok: if pcm_reservation is not None: pcm_reservation.rollback() _discard_retained_committed_audio() return False if actor.closing or runtime_closed or session.state != DuplexSessionState.OPEN: if pcm_reservation is not None: pcm_reservation.rollback() _discard_retained_committed_audio() return False if before_append is not None and not before_append(): if pcm_reservation is not None: pcm_reservation.rollback() _discard_retained_committed_audio() return True if pcm_reservation is not None and not pcm_reservation.active: _discard_retained_committed_audio() return False return await _run() def _release_cancelled_retained_audio(done: asyncio.Task[bool]) -> None: if done.cancelled(): _discard_retained_committed_audio() return try: append_ok = done.result() except Exception: append_ok = False if not append_ok: _discard_retained_committed_audio() predecessor = actor.native_append_tail if predecessor is not None and predecessor.done(): try: predecessor_ok = predecessor.result() except (asyncio.CancelledError, Exception): predecessor_ok = False if not predecessor_ok: # Appends already queued behind a failed predecessor keep # their captured dependency and stop. A later wire command # is an explicit retry and starts a new chain. predecessor = None task = asyncio.create_task(_run_in_wire_order(predecessor)) task.add_done_callback(_release_cancelled_retained_audio) actor.native_append_tail = task actor.track_append_task( task, epoch=append_epoch, mode="append_audio_chunk", final=final, response_bound=final or precreate_response, ) if silence_continuation: native.pending_silence_task = task def _clear_done_pending_silence(done: asyncio.Task[bool]) -> None: if native.pending_silence_task is done: native.pending_silence_task = None native.pending_silence_owner_id = None task.add_done_callback(_clear_done_pending_silence) # Let this wire-order effect start before the next mailbox event can # cancel it. Later native appends still serialize on predecessor. await asyncio.sleep(0) return task async def schedule_native_silence_continuation( payload: object, *, request_id: str, owner_id: str, response_id: str | None, response_owned: bool, expected_epoch: int | None, expected_incarnation: int, expected_model_turn_id: int | None, send_json, ) -> bool: del send_json if session is None: return False clear_completed_pending_silence() pending_silence = native.pending_silence_task if pending_silence is not None and not pending_silence.done(): if pending_silence is asyncio.current_task(): return False try: if not await pending_silence: return False except asyncio.CancelledError: current = asyncio.current_task() if current is not None and current.cancelling(): raise return False except Exception: return False clear_completed_pending_silence() pending_silence = native.pending_silence_task if pending_silence is not None and not pending_silence.done(): return False append_tail = actor.native_append_tail if (append_tail is None or append_tail.done()) and real_native_input_waiting(): return False continuation_delay_s = max( 0.0, float(session.capabilities.chunk_period_ms or 1000) / 1000.0, ) if continuation_delay_s > 0: await asyncio.sleep(continuation_delay_s) if ( actor.native_append_tail is not append_tail or ((append_tail is None or append_tail.done()) and real_native_input_waiting()) or self._native_silence_continuation_is_stale( session, request_id=request_id, response_id=response_id, response_owned=response_owned, expected_epoch=expected_epoch, expected_incarnation=expected_incarnation, expected_model_turn_id=expected_model_turn_id, ) ): return False def _still_valid() -> bool: return not real_native_input_waiting() and not self._native_silence_continuation_is_stale( session, request_id=request_id, response_id=response_id, response_owned=response_owned, expected_epoch=expected_epoch, expected_incarnation=expected_incarnation, expected_model_turn_id=expected_model_turn_id, ) native.pending_silence_owner_id = owner_id task = await start_native_append( payload, final=False, silence_continuation=True, before_append=_still_valid, ) if task is None: native.pending_silence_owner_id = None return False return True async def wait_for_native_append_tail() -> bool: predecessor = actor.native_append_tail if predecessor is None: return True try: return await predecessor except asyncio.CancelledError: current = asyncio.current_task() if current is not None and current.cancelling(): raise return False except Exception: # The append task already emitted its runtime error. Preserve # wire order before the update path decides whether to continue. return False async def start_runtime_append( payload: object, *, final: bool, mode: str = "append_tokens", ) -> None: """Schedule non-native runtime appends without blocking WS input. Model-native appends use ``start_native_append`` because they can create data-plane response streams. Generic runtime appends still need the same control-plane isolation so cancel/close can preempt a slow engine append. """ if session is None: return append_epoch = session.epoch async def _run() -> None: nonlocal runtime_closed try: append_ok, emitted_response = await self._append_runtime_input( session, payload, final=final, send_json=emit_event, mode=mode, expected_epoch=append_epoch, ) if not append_ok: if session.state == DuplexSessionState.CLOSED: runtime_closed = True return if not emitted_response and session.epoch == append_epoch: await emit_event(self._turn_controller.signal(session, DuplexTurnEventType.USER_STARTED.value)) except asyncio.CancelledError: raise except Exception as exc: logger.exception("Duplex runtime append task failed: %s", exc) await self._send_runtime_error(emit_event, "runtime_append_task_failed", exc, session=session) task = asyncio.create_task(_run()) actor.track_append_task( task, epoch=append_epoch, mode=mode, final=final, response_bound=final, ) try: handshake = await self._open_session( websocket, emit_event, realtime_protocol=realtime_protocol, attachment_send=attachment_send, attachment_close=attachment_close, ) if handshake is None: return session = handshake.session if initial_update_error := self._runtime_session_candidate_update_error( session, session.config, realtime_protocol=realtime_protocol ): await emit_event( correlate_realtime_error( initial_update_error, handshake.event_id, ) ) return if handshake.resumed: native = self._serving_session_states[session.session_id] actor.tasks = self._session_tasks[session.session_id] persisted_protocol = self._realtime_protocols.get(session.session_id) if persisted_protocol is None: raise RuntimeError(f"Missing Realtime protocol state for resumed session {session.session_id}") realtime_protocol = persisted_protocol realtime_protocol.bind_sender(send_realtime_raw) attachment_generation = handshake.attachment_generation attachment_ready = True resume_credential_delivered = True runtime_opened = True self._ensure_lifecycle_listener() reader_task = asyncio.create_task(read_event_loop(), name="duplex-session-reader") else: self._serving_session_states[session.session_id] = native self._session_tasks[session.session_id] = actor.tasks if realtime_protocol is not None: self._realtime_protocols[session.session_id] = realtime_protocol native.silence_continuation_scheduler = schedule_native_silence_continuation if realtime_protocol is not None: realtime_protocol.bind_native_input_append(self._uses_native_input_append(session)) session.config.playback_commit_policy = DuplexPlaybackCommitPolicy.ACK_ONLY.value if not handshake.resumed: open_result = await self._open_runtime_session(session, emit_event) if open_result is False: return runtime_opened = True try: await self._configure_server_vad(session) except Exception as exc: logger.exception("Server VAD initialization failed: %s", exc) self._realtime_vad_metrics.error("initialization") await emit_event( correlate_realtime_error( { "type": "error", "session_id": session.session_id, "code": "server_vad_initialization_failed", "error": str(exc), }, handshake.event_id, ) ) return created_attachment = await self._attachment_registry.create( session.session_id, incarnation=session.incarnation, send=attachment_send, close=attachment_close, ) attachment_generation = created_attachment.attachment_generation attachment_ready = True self._lease_generations[session.session_id] = 0 self._ensure_lifecycle_listener() created_payload: dict[str, object] = { "type": "session.created", "session": session.as_public_dict(), } if session.capabilities.supports_session_resume: created_payload.update( { "incarnation": session.incarnation, "attachment_generation": attachment_generation, "resume_token": created_attachment.resume_token.plaintext, } ) if isinstance(open_result, dict): created_payload["runtime_control"] = self._redact_runtime_control_result(open_result) if realtime_protocol is not None: realtime_protocol.commit_realtime_turn_detection_update(session.config.server_vad) await emit_event(created_payload) resume_credential_delivered = session.capabilities.supports_session_resume reader_task = asyncio.create_task(read_event_loop(), name="duplex-session-reader") while True: event = await next_actor_event() event_type = event.get("type") if event_type == "__replaced_attachment__": return if event_type == "__chat_response_finished__": completed_task = event.get("_response_task") if completed_task is not actor.active_response_task: continue actor.active_response_task = None activate_waiting_server_vad_turn() continue if event_type == "__timeout__": begin_close("timeout") native.audio_buffer.clear() session.release_all_input_bytes() native.input_since_commit = False native.speech_since_commit = False native.clear_committed_audio() await actor.cancel_append_tasks() await self._cancel_native_data_plane_stream(session) await self._cancel_active_response( session, actor.active_response_task, emit_event, reason="timeout", notify=True, ) actor.active_response_task = None runtime_closed = await self._close_runtime_session( session, reason="timeout", send_json=emit_event, ) if not runtime_closed: return session.close() await emit_event( { "type": "session.closed", "session_id": session.session_id, "reason": "timeout", } ) return if event_type == "__disconnect__": transport_detached = True return if not isinstance(event_type, str): continue if event_type == "session.heartbeat": touch_session = getattr(self._chat_service.engine_client, "touch_duplex_session_async", None) if not callable(touch_session): await emit_event( { "type": "error", "error": "Duplex runtime does not expose heartbeat control", "code": "runtime_touch_unsupported", } ) continue try: await touch_session( session.session_id, fence=DuplexFence( session.session_id, epoch=session.epoch, turn_id=session.turn_id, incarnation=session.incarnation, ), activity=DuplexLeaseActivity.HEARTBEAT, ) except Exception as exc: await emit_event( { "type": "error", "error": str(exc), "code": "runtime_touch_failed", } ) continue await emit_event( { "type": "session.heartbeat_ack", "session_id": session.session_id, } ) continue if event_type == "session.event_ack": acknowledged = event.get("server_event_seq") if not isinstance(acknowledged, int) or acknowledged < 0: await emit_event( { "type": "error", "error": "session.event_ack requires a non-negative server_event_seq", "code": "invalid_event_ack", } ) continue try: await self._attachment_registry.acknowledge(session.session_id, acknowledged) except ValueError as exc: await emit_event( { "type": "error", "error": str(exc), "code": "invalid_event_ack", } ) continue if event_type == "turn.signal" and event.get("event") in { "input.cancel", "response.cancel", "barge_in", }: payload = event.get("payload") normalized_event = dict(payload) if isinstance(payload, dict) else {} normalized_event.update(event) normalized_event["type"] = event["event"] event = normalized_event event_type = str(event["type"]) if event_type == "session.close": with suppress(asyncio.TimeoutError): await asyncio.wait_for(actor.output_queue.join(), timeout=1.0) begin_close("session_close") if session.state == DuplexSessionState.CLOSED: runtime_closed = True return native.audio_buffer.clear() session.release_all_input_bytes() native.input_since_commit = False native.speech_since_commit = False native.clear_committed_audio() await actor.cancel_append_tasks() await self._cancel_native_data_plane_stream(session) await self._cancel_active_response( session, actor.active_response_task, emit_event, reason="session_close", ) runtime_closed = await self._close_runtime_session( session, reason="session_close", send_json=emit_event, ) actor.active_response_task = None if not runtime_closed: return await emit_event({"type": "session.closed", "session_id": session.session_id}) session.close() return if event_type == "input_audio_buffer.clear": native.audio_buffer.clear() session.release_all_input_bytes() server_vad_pipeline = self._server_vad_pipelines.get(session.session_id) if server_vad_pipeline is not None: server_vad_pipeline.reset() native.input_since_commit = False native.speech_since_commit = False native.clear_committed_audio() cancelled = session.cancel_pending_input() await emit_event( { "type": "input_audio_buffer.cleared", "session_id": session.session_id, "epoch": session.epoch, "drained_input_events": 0, "cancelled": cancelled, } ) continue if event_type == "barge_in" and not session.capabilities.supports_barge_in: await emit_event(self._barge_in_unsupported_error(session)) continue if event_type in {"input.cancel", "response.cancel", "barge_in", "output_audio_buffer.clear"}: cancel_reason = ( "output_audio_buffer_clear" if event_type == "output_audio_buffer.clear" else "client_cancelled" if event_type == "response.cancel" else "barge_in" ) cancelled_fence = DuplexFence( session.session_id, epoch=session.epoch, turn_id=session.turn_id, incarnation=session.incarnation, ) if event_type == "response.cancel": requested_response_id = event.get("response_id") has_active_response_work = native_response_in_progress() if isinstance(requested_response_id, str) and ( session.active_response_id is None or requested_response_id != session.active_response_id ): await emit_event( { "type": "error", "session_id": session.session_id, "code": "response_not_active", "error": f"Response is not active: {requested_response_id}", } ) continue if event_type == "response.cancel": if not has_active_response_work: if realtime_protocol is not None and isinstance(requested_response_id, str): continue await emit_event( { "type": "error", "session_id": session.session_id, "code": "response_not_active", "error": "response.cancel requires an active response", } ) continue had_native_unbuffered_append = ( self._uses_native_input_append(session) and native.input_since_commit and not native.audio_buffer.has_pending() ) playback_was_active = self._assistant_playback_active(session) if event_type in {"input.cancel", "barge_in"}: native.audio_buffer.clear() session.release_all_input_bytes() native.input_since_commit = False native.speech_since_commit = False native.clear_committed_audio() had_native_append = await actor.cancel_append_tasks( response_bound_only=event_type in {"response.cancel", "output_audio_buffer.clear"}, ) if event_type == "response.cancel": session.release_input_bytes(native.clear_committed_audio()) had_native_stream = native.data_plane_task is not None cancelled = await self._cancel_active_response( session, actor.active_response_task, emit_event, reason=cancel_reason, ) had_native_stream = await self._cancel_native_data_plane_stream(session) or had_native_stream if not cancelled and (had_native_stream or had_native_append or had_native_unbuffered_append): old_epoch = session.epoch old_response_id = session.active_response_id committed_ms = session.playback.committed_ms self._commit_played_response_history(session, old_response_id, committed_ms) new_epoch, old_playback = self._advance_barge_in_epoch(session) await emit_event( { "type": "audio.cancelled", "session_id": session.session_id, "response_id": old_response_id, "reason": cancel_reason, "cancelled_epoch": old_epoch, "epoch": new_epoch, "committed_ms": committed_ms, "playback": old_playback, } ) cancelled = True if not cancelled and playback_was_active: old_epoch = session.epoch committed_ms = session.playback.committed_ms self._commit_played_response_history(session, session.last_response_id, committed_ms) new_epoch, old_playback = self._advance_barge_in_epoch(session) await emit_event( { "type": "audio.cancelled", "session_id": session.session_id, "response_id": session.last_response_id, "reason": cancel_reason, "cancelled_epoch": old_epoch, "epoch": new_epoch, "committed_ms": committed_ms, "playback": old_playback, } ) cancelled = True if not cancelled and event_type == "response.cancel": old_epoch = session.epoch old_response_id = session.active_response_id committed_ms = session.playback.committed_ms self._commit_played_response_history(session, old_response_id, committed_ms) new_epoch, old_playback = self._advance_barge_in_epoch(session) await emit_event( { "type": "audio.cancelled", "session_id": session.session_id, "response_id": old_response_id, "reason": cancel_reason, "cancelled_epoch": old_epoch, "epoch": new_epoch, "committed_ms": committed_ms, "playback": old_playback, } ) cancelled = True if not cancelled and event_type == "output_audio_buffer.clear": old_playback = session.playback.as_dict() committed_ms = session.playback.committed_ms session.clear_playback_cursor() await emit_event( { "type": "audio.cancelled", "session_id": session.session_id, "response_id": session.active_response_id, "reason": cancel_reason, "cancelled_epoch": session.epoch, "epoch": session.epoch, "committed_ms": committed_ms, "playback": old_playback, } ) actor.active_response_task = None activate_waiting_server_vad_turn() continue if not cancelled: await self._cancel_pending_input(session, emit_event, reason="barge_in") if not await self._signal_runtime_session( session, "barge_in", emit_event, fence=cancelled_fence, next_fence=( DuplexFence( session.session_id, epoch=session.epoch, turn_id=session.turn_id, incarnation=session.incarnation, ) if session.epoch > cancelled_fence.epoch else None ), ): continue actor.active_response_task = None activate_waiting_server_vad_turn() continue if event_type == "turn.signal": turn_event = event.get("event") if isinstance(turn_event, str): if turn_event == "barge_in" and not session.capabilities.supports_barge_in: await emit_event(self._barge_in_unsupported_error(session)) continue if turn_event == "session.update": realtime_event_id = event.get("realtime_event_id") async def emit_update_event(update_event: dict[str, object]) -> None: if isinstance(realtime_event_id, str) and realtime_event_id: update_event = {**update_event, "realtime_event_id": realtime_event_id} await emit_event(update_event) def reject_update() -> None: if realtime_protocol is not None: realtime_protocol.reject_realtime_turn_detection_update() payload = event.get("payload") if not isinstance(payload, dict): await emit_update_event( { "type": "error", "session_id": session.session_id, "code": "bad_event", "error": "session.update requires a session payload", } ) reject_update() continue if not await wait_for_native_append_tail(): await emit_update_event( { "type": "error", "code": "session_update_aborted", "error": "session.update was not applied because the preceding append failed", } ) reject_update() continue if ( realtime_protocol is not None and not self._uses_native_input_append(session) and session.config.server_vad is not None and session.turn_detection_config_locked and realtime_protocol.pending_realtime_input_audio_config_changed() ): await emit_update_event( { "type": "error", "session_id": session.session_id, "code": "unsupported_audio_format", "error": ( "session.update cannot change Server VAD input format or sample rate " "after the first audio append" ), "param": "audio.input.format", } ) reject_update() continue runtime_update_error = self._runtime_session_update_error(session, payload) if runtime_update_error is not None: await emit_update_event(runtime_update_error) reject_update() continue previous_config = session.config candidate_config = deepcopy(previous_config) session.replace_config(candidate_config) try: update_error = self._apply_session_update(session, payload) finally: session.replace_config(previous_config) server_vad_changed = candidate_config.server_vad != previous_config.server_vad if update_error is not None: await emit_update_event(update_error) reject_update() continue runtime_update_error = self._runtime_session_candidate_update_error( session, candidate_config, realtime_protocol=realtime_protocol, ) if runtime_update_error is not None: await emit_update_event(runtime_update_error) reject_update() continue try: candidate_runtime_config = self._runtime_config_for_session_update( session, candidate_config, ) except ServingRuntimeConfigError as exc: await emit_update_event( { "type": "error", "session_id": session.session_id, "code": exc.code, "error": str(exc), } ) reject_update() continue candidate_server_vad_pipeline = None if server_vad_changed: try: candidate_server_vad_pipeline = await self._prepare_server_vad_pipeline( session, candidate_config, ) except Exception as exc: logger.exception("Server VAD session update failed: %s", exc) self._realtime_vad_metrics.error("initialization") await emit_update_event( { "type": "error", "session_id": session.session_id, "code": "server_vad_initialization_failed", "error": str(exc), } ) reject_update() continue runtime_updated = await self._signal_runtime_session( session, turn_event, emit_update_event, session_config=candidate_config.as_dict(), runtime_config=candidate_runtime_config, ) if not runtime_updated: reject_update() continue session.replace_config(candidate_config) session.replace_runtime_config(candidate_runtime_config) if server_vad_changed: self._install_server_vad_pipeline( session, candidate_server_vad_pipeline, ) if realtime_protocol is not None: realtime_protocol.bind_native_input_append(self._uses_native_input_append(session)) realtime_protocol.commit_realtime_turn_detection_update(session.config.server_vad) await emit_update_event( { "type": "session.updated", "session": session.as_public_dict(), } ) continue if turn_event == "conversation.item.create": payload = event.get("payload") item = payload.get("item") if isinstance(payload, dict) else None item_type = item.get("type") if isinstance(item, dict) else None prepare_function_output = getattr( self._serving_runtime_adapter, "runtime_config_for_function_output", None, ) if item_type == "function_call_output" and callable(prepare_function_output): if not await wait_for_native_append_tail(): continue try: candidate_runtime_config = prepare_function_output( item, dict(session.runtime_config), ) except ServingRuntimeConfigError as exc: await emit_event( { "type": "error", "session_id": session.session_id, "code": exc.code, "error": str(exc), } ) continue if not await self._signal_runtime_session( session, "session.update", emit_event, runtime_config=candidate_runtime_config, ): continue session.replace_runtime_config(candidate_runtime_config) await emit_event( { "type": "conversation.item.created", "session_id": session.session_id, "item": item, "created": True, } ) await self._maybe_continue_native_response( emit_event, session=session, expected_epoch=session.epoch, expected_model_turn_id=session.turn_id, ) continue history_item = payload.get("history_item") if isinstance(payload, dict) else None if not isinstance(history_item, dict): history_item = item message = self._realtime_item_to_history_message(history_item) item_id = item.get("id") if isinstance(item, dict) else None if message is not None: session.append_history_message(message) session.register_history_item(item_id if isinstance(item_id, str) else None, message) await emit_event( { "type": "conversation.item.created", "session_id": session.session_id, "item": item, "created": message is not None, } ) continue if turn_event == "conversation.item.delete": payload = event.get("payload") item_id = payload.get("item_id") if isinstance(payload, dict) else None deleted = session.delete_history_item(item_id) if isinstance(item_id, str) else False await emit_event( { "type": "conversation.item.deleted", "session_id": session.session_id, "item_id": item_id, "deleted": deleted, } ) continue if turn_event == "conversation.item.truncate": payload = event.get("payload") item_id = payload.get("item_id") if isinstance(payload, dict) else None audio_end_ms = payload.get("audio_end_ms") if isinstance(payload, dict) else None truncated = ( session.truncate_history_item( item_id, audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0, hard=True, ) if isinstance(item_id, str) else False ) await emit_event( { "type": "conversation.item.truncated", "session_id": session.session_id, "item_id": item_id, "content_index": ( payload.get("content_index", 0) if isinstance(payload, dict) else 0 ), "audio_end_ms": audio_end_ms, "truncated": truncated, } ) continue await emit_event(self._turn_controller.signal(session, turn_event, event)) else: await emit_event({"type": "error", "error": "turn.signal requires event", "code": "bad_event"}) continue if event_type == "playback.ack": await self._handle_playback_ack(session, event, emit_event) continue if event_type == "input.text.append": session.mark_user_input_activity() text = event.get("text") if not isinstance(text, str): await emit_event( { "type": "error", "error": "input.text.append requires text", "code": "bad_event", } ) continue if self._uses_native_input_append(session): await emit_event( { "type": "error", "error": "The selected native duplex runtime accepts audio append only", "code": "native_text_append_unsupported", } ) continue else: session.append_text(text) if session.capabilities.supports_input_append: await start_runtime_append(text, final=False, mode="append_tokens") continue if event_type == "input_audio_buffer.append": session.mark_user_input_activity() audio = event.get("audio") or event.get("data") if not isinstance(audio, str): await emit_event( { "type": "error", "error": "input.audio.append requires audio", "code": "bad_event", } ) continue if not session.capabilities.supports_barge_in and self._event_requests_barge_in(event): await emit_event(self._barge_in_unsupported_error(session)) event = dict(event) event.pop("force_barge_in", None) for key in ("overlap_action", "overlap"): value = event.get(key) if isinstance(value, str) and value.strip().lower() in { "barge_in", "interrupt", "cancel", }: event.pop(key, None) fmt = event.get("format") if isinstance(event.get("format"), str) else "pcm16" default_sample_rate_hz = 16000 sr_raw = event.get("sample_rate_hz") or event.get("sample_rate") sample_rate_hz = sr_raw if isinstance(sr_raw, int | float) else default_sample_rate_hz native_input = self._uses_native_input_append(session) server_vad_config = session.config.server_vad turn_based_server_vad = server_vad_config is not None and not native_input if not turn_based_server_vad: try: audio, fmt, sample_rate_hz = convert_input_audio_with_rate( audio, fmt, sample_rate_hz=sample_rate_hz, ) except ValueError as exc: await emit_event({"type": "error", "error": str(exc), "code": "bad_event"}) continue if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le"}: await emit_event( { "type": "error", "error": "input_audio_buffer.append pcm16 audio could not be decoded", "code": "bad_audio", } ) continue if server_vad_config is None: session.lock_turn_detection_config() force_listen = bool(event.get("force_listen", False)) payload = { "type": "audio", "audio": audio, "format": fmt, "sample_rate_hz": sample_rate_hz, "force_listen": force_listen, } video_frames = event.get("video_frames") if isinstance(video_frames, list): frames = [frame for frame in video_frames if isinstance(frame, str) and frame] if frames: payload["video_frames"] = frames if server_vad_config is not None: realtime_event_id = event.get("_realtime_event_id") pipeline = self._server_vad_pipelines.get(session.session_id) if pipeline is None: await emit_event( correlate_realtime_error( { "type": "error", "session_id": session.session_id, "code": "internal_error", "error": "server_vad is configured but its detector pipeline is unavailable", }, realtime_event_id, ) ) continue speech_was_active = pipeline.speech_active reserved_bytes = 0 previous_scratch_bytes = pipeline.scratch_bytes try: if native_input: if fmt != "pcm_f32le" or sample_rate_hz != pipeline.sample_rate_hz: raise ValueError("native server_vad input must be mono PCM float32 at 16000 Hz") raw_audio = base64.b64decode(audio, validate=True) if len(raw_audio) % np.dtype(" 0 else: if fmt != "pcm16": raise ValueError("server_vad input must be mono PCM16") source_sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz) if source_sample_rate_hz not in {16_000, 24_000}: raise ValueError("server_vad PCM16 input sample rate must be 16000 or 24000 Hz") if ( pipeline.source_sample_rate_hz is not None and source_sample_rate_hz != pipeline.source_sample_rate_hz ): raise ValueError( "server_vad input sample rate cannot change within a continuous audio stream" ) try: raw_audio = base64.b64decode(audio, validate=True) except (binascii.Error, ValueError) as exc: raise ValueError( "server_vad input audio must be valid base64-encoded PCM16" ) from exc if len(raw_audio) % np.dtype(" 0 except (binascii.Error, ValueError) as exc: await emit_event( correlate_realtime_error( { "type": "error", "session_id": session.session_id, "code": "bad_audio", "error": str(exc), "param": "audio", }, realtime_event_id, ) ) continue if not native_input: normalized_sample_upper_bound = ( source_samples if source_sample_rate_hz == pipeline.sample_rate_hz else ((source_samples + 2) // 3) * 2 ) reserved_bytes = normalized_sample_upper_bound * np.dtype(np.float32).itemsize if not session.reserve_input_bytes( reserved_bytes, limit=self._duplex_session_config.max_pending_input_bytes_per_session, ): await reject_and_clear_server_vad_input( pipeline, code="input_backpressure", error="Server VAD input exceeds the per-session input limit", metric_reason="input_backpressure", event_id=realtime_event_id, ) continue try: vad_batch = ( await pipeline.push(normalized_audio) if native_input else await pipeline.push_pcm16( raw_audio, source_sample_rate_hz=source_sample_rate_hz, ) ) except Exception as exc: logger.exception("Server VAD inference failed: %s", exc) await reject_and_clear_server_vad_input( pipeline, code="server_vad_inference_failed", error=str(exc), metric_reason="inference", event_id=realtime_event_id, ) continue if not native_input: retained_delta = ( sum(frame.samples.nbytes for frame in vad_batch.frames) + pipeline.scratch_bytes - previous_scratch_bytes ) session.release_input_bytes(max(0, reserved_bytes - retained_delta)) self._realtime_vad_metrics.observe_inference(vad_batch.inference_ms) prefix_samples = round(server_vad_config.prefix_padding_ms * pipeline.sample_rate_hz / 1000) speech_started = False speech_stopped = False for frame_index, vad_frame in enumerate(vad_batch.frames): decision = vad_frame.decision if not native_input: released_bytes = session.append_server_vad_frame( vad_frame.samples, speech_started=decision.speech_started, speech_stopped=decision.speech_stopped, prefix_samples=prefix_samples, ) session.release_input_bytes(released_bytes) if decision.speech_started: speech_started = True item_id = f"item_{uuid.uuid4().hex}" session.begin_server_vad_speech(item_id) await emit_event( { "type": "input_audio_buffer.speech_started", "session_id": session.session_id, "item_id": item_id, "audio_start_ms": decision.audio_start_ms or 0, } ) if decision.speech_stopped: speech_stopped = True item_id = session.finish_server_vad_speech() or f"item_{uuid.uuid4().hex}" await emit_event( { "type": "input_audio_buffer.speech_stopped", "session_id": session.session_id, "item_id": item_id, "audio_end_ms": decision.audio_end_ms or 0, } ) if decision.endpoint_delay_ms is not None: self._realtime_vad_metrics.observe_endpoint_delay(decision.endpoint_delay_ms) if not native_input: remaining_frame_bytes = sum( frame.samples.nbytes for frame in vad_batch.frames[frame_index + 1 :] ) await commit_server_vad_turn( item_id=item_id, create_response=server_vad_config.create_response, retain_input_bytes=remaining_frame_bytes + pipeline.scratch_bytes, event_id=realtime_event_id, ) if accepted_audio: session.lock_turn_detection_config() if not native_input: continue speech_active = pipeline.speech_active contained_speech = speech_was_active or speech_started or speech_stopped or speech_active payload["is_speech"] = contained_speech payload["vad"] = { "backend": "silero", "is_speech": contained_speech, "speech_active": speech_active, "speech_started": speech_started, "speech_stopped": speech_stopped, } if speech_active: payload["force_listen"] = True # Speech/silence tag for the Stage0 turn-ended latch. payload["is_speech"] = self._input_looks_like_speech(event, payload, session=session) defer_native_append = False buffer_overlap_audio = True if native_input: mark_pending_silence_superseded() overlap_active = native_response_in_progress() and ( not self._session_auto_responds(session) or ( session.capabilities.supports_barge_in and ( session.config.overlap_policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value or self._event_requests_barge_in(event) ) ) ) if overlap_active: decision = self._overlap_decision(session, event, payload) await self._emit_overlap_decision(emit_event, session, decision) action = decision.get("action") if action == "drop": if realtime_protocol is not None: await realtime_protocol.discard_pending_input_audio( audio_end_ms=self._input_audio_duration_ms(event, payload) ) continue if action == "listen": buffer_overlap_audio = bool(decision.get("buffer_audio", True)) defer_native_append = bool(decision.get("defer_runtime_append", True)) if ( not buffer_overlap_audio and realtime_protocol is not None and decision.get("preserve_realtime_input") is not True ): await realtime_protocol.discard_pending_input_audio( audio_end_ms=self._input_audio_duration_ms(event, payload) ) if decision.get("force_listen", True) is True: payload["force_listen"] = True else: event["force_barge_in"] = True cancelled_fence = DuplexFence( session.session_id, epoch=session.epoch, turn_id=session.turn_id, incarnation=session.incarnation, ) playback_was_active = self._assistant_playback_active(session) buffer_overlap_audio = True defer_native_append = False native.audio_buffer.clear_force_listen() session.reset_overlap_speech() native.input_since_commit = False native.speech_since_commit = False await actor.cancel_append_tasks() had_native_stream = native.data_plane_task is not None cancel_reason = str(decision.get("cancel_reason") or "barge_in") cancelled = await self._cancel_active_response( session, actor.active_response_task, emit_event, reason=cancel_reason, ) had_native_stream = ( await self._cancel_native_data_plane_stream(session) or had_native_stream ) if not cancelled and had_native_stream: old_epoch = session.epoch old_response_id = session.active_response_id committed_ms = session.playback.committed_ms self._commit_played_response_history(session, old_response_id, committed_ms) new_epoch, old_playback = self._advance_barge_in_epoch(session) await emit_event( { "type": "audio.cancelled", "session_id": session.session_id, "response_id": old_response_id, "reason": cancel_reason, "cancelled_epoch": old_epoch, "epoch": new_epoch, "committed_ms": committed_ms, "playback": old_playback, } ) cancelled = True if not cancelled and playback_was_active: old_epoch = session.epoch committed_ms = session.playback.committed_ms self._commit_played_response_history( session, session.last_response_id, committed_ms ) new_epoch, old_playback = self._advance_barge_in_epoch(session) await emit_event( { "type": "audio.cancelled", "session_id": session.session_id, "response_id": session.last_response_id, "reason": cancel_reason, "cancelled_epoch": old_epoch, "epoch": new_epoch, "committed_ms": committed_ms, "playback": old_playback, } ) cancelled = True if session.epoch > cancelled_fence.epoch: if not await self._signal_runtime_session( session, "barge_in", emit_event, fence=cancelled_fence, next_fence=DuplexFence( session.session_id, epoch=session.epoch, turn_id=session.turn_id, incarnation=session.incarnation, ), ): continue actor.active_response_task = None elif not self._session_auto_responds(session) and not self._input_looks_like_speech( event, payload, session=session ): # Turn-mode only: skip silent chunks so they don't open a # response. In auto-respond (full-duplex) mode the model owns # the speak/listen decision and MUST receive silence units -- # the official model typically starts speaking during the # silence after a question. # No response_id: a silence-skip listen can arrive # while a response is still streaming and must not # read as that response's terminal decision. await emit_event( { "type": "response.listen", "session_id": session.session_id, "epoch": session.epoch, "reason": "silence_or_noise", } ) continue if self._should_force_listen_for_auto_response_overlap(session, event, payload): # Auto-response keeps a long-lived native Stage0 stream. # While assistant audio is still active, silence from the # browser should advance the model in listen mode rather # than letting the same stream start another assistant # segment. Speech/barge-in chunks remain model-owned. payload["force_listen"] = True if not buffer_overlap_audio: continue session.mark_user_input_activity() native.input_since_commit = True native.speech_since_commit = native.speech_since_commit or self._input_looks_like_speech( event, payload, session=session ) try: raw_audio_bytes = self._native_audio_payload_size_bytes(payload) if not session.reserve_input_bytes( raw_audio_bytes, limit=self._duplex_session_config.max_pending_input_bytes_per_session, ): await emit_event( { "type": "error", "error": "Duplex session pending input exceeds server limit", "code": "input_backpressure", } ) continue allow_emit = not defer_native_append and ( realtime_protocol is None or event_type != "input_audio_buffer.append" # Full-duplex: emit each ~chunk_period of audio so the model # runs per-chunk generation (speak/listen) without an explicit # response.create, matching the official duplex_generate loop. or self._session_auto_responds(session) ) pcm_reservation = native.audio_buffer.prepare_append( payload, operation_id=uuid.uuid4().hex, chunk_period_ms=session.capabilities.chunk_period_ms or 1000, allow_emit=allow_emit, ) except ValueError as exc: session.release_input_bytes(raw_audio_bytes) await emit_event({"type": "error", "error": str(exc), "code": "bad_event"}) continue if pcm_reservation is None: continue if pcm_reservation.byte_count == 0: session.release_input_bytes(raw_audio_bytes) payload = pcm_reservation.payload else: session.append_audio(audio, fmt=fmt, sample_rate_hz=sample_rate_hz) if native_input: await start_native_append( payload, final=False, pcm_reservation=pcm_reservation, ) continue if session.capabilities.supports_input_append: await start_runtime_append(payload, final=False, mode="append_audio_chunk") continue if event_type in {"input.commit", "input_audio_buffer.commit", "response.create"}: if ( event_type == "response.create" and session.config.server_vad is not None and actor.active_response_task is not None ): await emit_event( { "type": "error", "session_id": session.session_id, "code": "response_already_active", "error": "response.create must wait for the active response to complete", } ) continue item_id = event.get("item_id") realtime_validated_audio_commit = ( event_type == "input_audio_buffer.commit" and isinstance(item_id, str) and bool(item_id) ) if ( self._uses_native_input_append(session) and event_type in {"input.commit", "input_audio_buffer.commit"} and not await wait_for_native_append_tail() ): continue if event_type == "input_audio_buffer.commit" and event.get("is_speech") is False: native.input_since_commit = False native.speech_since_commit = False native.audio_buffer.clear() session.release_all_input_bytes() native.clear_committed_audio() await emit_event( { "type": "input.committed", "session_id": session.session_id, "item_id": item_id, "turn_id": session.turn_id, "epoch": session.epoch, "empty": True, "is_speech": False, "no_response": True, } ) # No response_id: this silence listen answers an empty # commit, it is not a response terminal. await emit_event( { "type": "response.listen", "session_id": session.session_id, "epoch": session.epoch, "reason": "silence_or_noise", } ) continue should_create_response = ( event_type == "response.create" or bool(event.get("response_create", event_type == "input.commit")) or (event_type == "input_audio_buffer.commit" and self._session_auto_responds(session)) ) precreate_response_requested = event_type == "response.create" or bool( event.get("response_create", event_type == "input.commit") ) if event_type == "response.create": response_payload = event.get("response") if isinstance(response_payload, dict): response_options_error = self._apply_response_create_options(session, response_payload) if response_options_error is not None: error_message = ( "The selected native duplex runtime does not support generation " "overrides for instructions, voice, temperature, max tokens, tools, or " "tool_choice." if response_options_error == "unsupported_native_response_options" else "response.create cannot reserve options while another response is active." ) await emit_event( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": response_options_error, "error": error_message, } ) continue if self._uses_native_input_append(session) and event_type == "input_audio_buffer.commit": has_pending_native_audio = ( native.input_since_commit or native.audio_buffer.has_pending() or native.committed_audio_payload is not None or realtime_validated_audio_commit ) if ( not has_pending_native_audio and not actor.native_append_tasks and native.data_plane_task is None ): await emit_event( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": "input_audio_buffer_empty", "error": "input_audio_buffer.commit requires a non-empty input audio buffer.", } ) continue commit_action = decide_commit_action( CommitSnapshot( auto_responds=self._session_auto_responds(session), speech_since_commit=native.speech_since_commit, active_response_id=session.active_response_id, overlap_speech_ms=session.overlap_speech_ms, native_response_in_progress=native_response_in_progress(), playback_active=self._assistant_playback_active(session), ) ) if commit_action is CommitAction.DEFER_ACTIVE_RESPONSE: if session.overlap_speech_ms <= session.config.overlap_short_ack_ms: native.audio_buffer.clear() session.release_all_input_bytes() native.input_since_commit = False native.speech_since_commit = False native.clear_committed_audio() if realtime_protocol is not None: await realtime_protocol.discard_pending_input_audio( audio_end_ms=session.overlap_speech_ms ) await emit_event( { "type": "input.committed", "session_id": session.session_id, "item_id": item_id, "turn_id": session.turn_id, "epoch": session.epoch, "empty": True, "is_speech": False, "overlap_ack": True, "no_response": True, } ) session.reset_overlap_speech() session.discard_response_options() continue commit_reservation = native.audio_buffer.prepare_commit( operation_id=uuid.uuid4().hex, chunk_period_ms=session.capabilities.chunk_period_ms or 1000, ) deferred_payload = commit_reservation.payload if deferred_payload is None: commit_reservation.commit() else: if native.committed_audio_payload is not None: deferred_payload = self._merge_native_audio_payloads( native.committed_audio_payload, deferred_payload, ) native.retain_committed_audio( deferred_payload, operation_id=commit_reservation.operation_id, reserved_bytes=commit_reservation.byte_count, ) commit_reservation.commit() native.deferred_response_create = should_create_response native.deferred_precreate_response = precreate_response_requested native.input_since_commit = False native.speech_since_commit = False committed = self._commit_native_audio_input( session, item_id=event.get("item_id"), transcript=event.get("transcript"), ) committed_payload = self._native_audio_committed_payload( session, committed=committed, item_id=event.get("item_id"), transcript=event.get("transcript"), ) committed_payload["overlap_deferred"] = True committed_payload["response_create_deferred"] = should_create_response await emit_event(committed_payload) continue if commit_action is CommitAction.START_AUTO_RESPONSE: commit_reservation = native.audio_buffer.prepare_commit( operation_id=uuid.uuid4().hex, chunk_period_ms=session.capabilities.chunk_period_ms or 1000, ) committed_input = commit_reservation.payload final_payload = committed_input if native.committed_audio_payload is not None: if final_payload is not None: final_payload = self._merge_native_audio_payloads( native.committed_audio_payload, final_payload, ) else: final_payload = native.committed_audio_payload commit_reservation.commit() if final_payload is not None: native.retain_committed_audio( final_payload, operation_id=commit_reservation.operation_id, reserved_bytes=commit_reservation.byte_count, ) native.deferred_response_create = False native.input_since_commit = False native.speech_since_commit = False data_plane_turn_id = session.turn_id committed = self._commit_native_audio_input( session, item_id=event.get("item_id"), transcript=event.get("transcript"), turn_id=data_plane_turn_id, ) await emit_event( self._native_audio_committed_payload( session, committed=committed, item_id=event.get("item_id"), transcript=event.get("transcript"), ) ) if final_payload is not None: await start_native_append( { **final_payload, "duplex_turn_id": data_plane_turn_id, }, final=True, precreate_response=False, operation_id=commit_reservation.operation_id, retained_committed_payload=final_payload, ) continue if self._uses_native_input_append(session) and event_type == "response.create": if ( native_response_in_progress() or actor.native_append_tasks or native.data_plane_task is not None ): if session.active_response_id is None and ( session.active_request_id is not None or actor.native_append_tasks or native.data_plane_task is not None ): continue await emit_event( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": "response_already_active", "error": "response.create cannot start while another response is active.", } ) session.discard_response_options() continue if native.committed_audio_payload is not None: committed_payload = native.committed_audio_payload operation_id = native.committed_audio_operation_id if operation_id is None: operation_id = uuid.uuid4().hex native.committed_audio_operation_id = operation_id await start_native_append( committed_payload, final=True, precreate_response=True, operation_id=operation_id, retained_committed_payload=committed_payload, ) continue await emit_event( { "type": "error", "session_id": session.session_id, "epoch": session.epoch, "code": "response_create_without_input", "error": "Native duplex response.create requires committed audio input.", } ) session.discard_response_options() continue if self._uses_native_input_append(session) and not native_response_in_progress(): commit_reservation = ( native.audio_buffer.prepare_commit( operation_id=uuid.uuid4().hex, chunk_period_ms=session.capabilities.chunk_period_ms or 1000, ) if event_type in {"input.commit", "input_audio_buffer.commit"} else None ) flushed_buffer_reserved_bytes = ( native.audio_buffer.pending_byte_count if commit_reservation is None else 0 ) flushed = ( commit_reservation.payload if commit_reservation is not None else native.audio_buffer.flush(chunk_period_ms=session.capabilities.chunk_period_ms or 1000) ) if native.committed_audio_payload is not None: if flushed is not None: flushed = self._merge_native_audio_payloads( native.committed_audio_payload, flushed, ) else: flushed = native.committed_audio_payload if commit_reservation is not None: commit_reservation.commit() if flushed is not None: if self._should_force_listen_for_short_commit(session, event, flushed): flushed = dict(flushed) flushed["force_listen"] = True native.input_since_commit = False committed = self._commit_native_audio_input( session, item_id=event.get("item_id"), transcript=event.get("transcript"), ) await emit_event( self._native_audio_committed_payload( session, committed=committed, item_id=event.get("item_id"), transcript=event.get("transcript"), ) ) if should_create_response: native.retain_committed_audio( flushed, operation_id=( commit_reservation.operation_id if commit_reservation is not None else uuid.uuid4().hex ), reserved_bytes=( commit_reservation.byte_count if commit_reservation is not None else flushed_buffer_reserved_bytes ), ) await start_native_append( flushed, final=True, precreate_response=True, operation_id=native.committed_audio_operation_id, retained_committed_payload=flushed, ) else: native.retain_committed_audio( flushed, operation_id=( commit_reservation.operation_id if commit_reservation is not None else uuid.uuid4().hex ), reserved_bytes=( commit_reservation.byte_count if commit_reservation is not None else flushed_buffer_reserved_bytes ), ) native.deferred_response_create = False continue native_had_uncommitted_audio = self._uses_native_input_append(session) and ( native.input_since_commit or native.audio_buffer.has_pending() or native.committed_audio_payload is not None or realtime_validated_audio_commit ) committed = session.commit_user_input() if self._uses_native_input_append(session) and event_type in { "input_audio_buffer.commit", "input.commit", }: native.input_since_commit = False native.speech_since_commit = False if committed is None and event_type != "response.create": if self._uses_native_input_append(session): committed = ( self._commit_native_audio_input( session, item_id=event.get("item_id"), transcript=event.get("transcript"), ) if native_had_uncommitted_audio else None ) await emit_event( self._native_audio_committed_payload( session, committed=committed, item_id=event.get("item_id"), transcript=event.get("transcript"), ) ) else: await emit_event( { "type": "input.committed", "session_id": session.session_id, "item_id": item_id, "empty": True, } ) continue if committed is not None: item_id = event.get("item_id") if isinstance(item_id, str): session.register_history_item(item_id, committed.message) await emit_event( self._input_committed_payload( session, committed, item_id=item_id, ) ) if not should_create_response: continue if actor.active_response_task is not None and not actor.active_response_task.done(): await self._cancel_active_response( session, actor.active_response_task, emit_event, reason="new_response", ) if session.config.server_vad is not None and not self._uses_native_input_append(session): start_chat_response() else: actor.active_response_task = asyncio.create_task(self._run_response(session, emit_event)) continue await emit_event( { "type": "error", "error": f"Unknown duplex event: {event_type}", "code": "unknown_event", } ) except WebSocketDisconnect: logger.info("Duplex session disconnected") transport_detached = True except Exception as exc: logger.exception("Duplex session failed: %s", exc) with suppress(Exception): await emit_event({"type": "error", "error": str(exc), "code": "internal_error"}) finally: if realtime_protocol is not None: realtime_protocol.reject_realtime_turn_detection_update() if session is not None: if reader_task is not None and not reader_task.done(): reader_task.cancel() with suppress(asyncio.CancelledError): await reader_task while pending_turn_reservations > 0: session.release_pending_turn() pending_turn_reservations -= 1 resumable_detach = ( transport_detached and runtime_opened and not runtime_closed and session.state == DuplexSessionState.OPEN and session.capabilities.supports_session_resume and attachment_generation is not None and resume_credential_delivered ) if resumable_detach: async def cancel_orphan_response_after_grace() -> None: tasks = self._session_tasks.get(session.session_id) current_session = self._registry.get(session.session_id) if tasks is None or current_session is not session or session.state != DuplexSessionState.OPEN: return await tasks.cancel_append_tasks(response_bound_only=True) await self._cancel_native_data_plane_stream(session) await self._cancel_active_response( session, tasks.active_response_task, emit_event, reason="disconnect_grace_expired", notify=False, ) tasks.active_response_task = None detached = await self._attachment_registry.detach( session.session_id, attachment_generation=attachment_generation, on_grace_expired=cancel_orphan_response_after_grace, ) touch_session = getattr(self._chat_service.engine_client, "touch_duplex_session_async", None) if detached and callable(touch_session): with suppress(Exception): await touch_session( session.session_id, fence=DuplexFence( session.session_id, epoch=session.epoch, turn_id=session.turn_id, incarnation=session.incarnation, ), activity=DuplexLeaseActivity.DETACH, ) else: begin_close(actor.close_reason or "disconnect") await actor.cancel_append_tasks() await self._cancel_native_data_plane_stream(session) await self._cancel_active_response( session, actor.active_response_task, emit_event, reason="disconnect", notify=False, ) if runtime_opened and not runtime_closed and session.state != DuplexSessionState.CLOSED: await self._close_runtime_session(session, reason="disconnect") self._cleanup_duplex_session_state(session) self._registry.close(session.session_id) self._session_tasks.pop(session.session_id, None) self._realtime_protocols.pop(session.session_id, None) self._lease_generations.pop(session.session_id, None) self._resync_required_sessions.discard(session.session_id) with suppress(Exception): await self._attachment_registry.close(session.session_id) self._stop_lifecycle_listener_if_idle() await actor.close_writer() with suppress(Exception): await asyncio.wait_for(actor.output_queue.join(), timeout=2.0) if not writer_task.done(): writer_task.cancel() with suppress(asyncio.CancelledError): await writer_task