anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity
Files
4a8ac297751b03fbe21f2d8a4550a027fe02e7cd
vllm-omni/vllm_omni/entrypoints/duplex/session_runner.py
T

2521 lines
129 KiB
Python
Raw Blame History

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import asyncio
import binascii
import json
import time
import uuid
from collections.abc import Callable
from contextlib import suppress
from copy import deepcopy
import numpy as np
import pybase64 as base64
from fastapi import WebSocket, WebSocketDisconnect
from vllm.logger import init_logger
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
from vllm_omni.engine.duplex.messages import DuplexFence
from vllm_omni.entrypoints.duplex.audio import (
convert_input_audio_with_rate,
validate_input_sample_rate_hz,
)
from vllm_omni.entrypoints.duplex.commit_policy import (
CommitAction,
CommitSnapshot,
decide_commit_action,
)
from vllm_omni.entrypoints.duplex.protocol import (
DuplexOverlapPolicy,
DuplexPlaybackCommitPolicy,
DuplexSession,
DuplexSessionState,
DuplexTurnEventType,
)
from vllm_omni.entrypoints.duplex.realtime_session import (
NativeRealtimeSessionProtocol,
)
from vllm_omni.entrypoints.duplex.runtime_adapter import (
PcmAppendReservation,
ServingRuntimeConfigError,
ServingRuntimeSessionState,
payload_turn_id,
)
from vllm_omni.entrypoints.duplex.server_vad import ServerVADPipeline
from vllm_omni.entrypoints.duplex.session_attachment import (
DuplexJournalOverflowError,
)
from vllm_omni.entrypoints.duplex.websocket import (
DuplexWebSocketActor,
is_input_event,
normalize_duplex_input_event,
)
logger = init_logger(__name__)
_MAX_EVENT_BYTES = 15 * 1024 * 1024
class DuplexSessionRunnerMixin:
"""Run one ordered WebSocket mailbox for a duplex session."""
async def handle_session(
self,
websocket: WebSocket,
*,
realtime_protocol: NativeRealtimeSessionProtocol | None = None,
) -> None:
await websocket.accept()
session: DuplexSession | None = None
actor = DuplexWebSocketActor(
websocket,
current_epoch=lambda: session.epoch if session is not None else None,
session_closed=lambda: session is not None and session.state == DuplexSessionState.CLOSED,
outbound_protocol=realtime_protocol,
)
runtime_opened = False
runtime_closed = False
attachment_ready = False
attachment_generation: int | None = None
resume_credential_delivered = False
transport_detached = False
pending_turn_reservations = 0
async def attachment_send(payload: dict[str, object]) -> None:
nonlocal resume_credential_delivered
try:
await websocket.send_json(payload)
except RuntimeError as exc:
message = str(exc)
if "after sending 'websocket.close'" in message or "response already completed" in message:
raise WebSocketDisconnect(code=1006) from exc
raise
if payload.get("type") == "session.created" and isinstance(payload.get("resume_token"), str):
resume_credential_delivered = True
async def attachment_close(reason: str) -> None:
close = getattr(websocket, "close", None)
if callable(close):
await close(code=1000, reason=reason)
async def send_attachment_payload(
payload: dict[str, object], *, journal: bool, on_accepted: Callable[[], None] | None = None
) -> None:
assert session is not None
session_id = session.session_id
should_journal = journal and session_id not in self._resync_required_sessions
try:
await self._attachment_registry.send_event(
session_id,
payload,
journal=should_journal,
on_accepted=on_accepted,
)
except DuplexJournalOverflowError:
first_overflow = session_id not in self._resync_required_sessions
self._resync_required_sessions.add(session_id)
if first_overflow:
resync_payload: dict[str, object] = {
"type": "session.resync_required",
"session_id": session_id,
"reason": "journal_overflow",
}
if realtime_protocol is not None:
resync_payload = realtime_protocol.encode_outbound_event(resync_payload)[0]
await self._attachment_registry.send_event(
session_id,
resync_payload,
journal=False,
)
await self._attachment_registry.send_event(
session_id,
payload,
journal=False,
on_accepted=on_accepted,
)
if realtime_protocol is not None:
async def send_realtime_raw(payload: dict[str, object]) -> None:
if attachment_ready and session is not None:
await send_attachment_payload(
payload,
journal=payload.get("type") not in {"session.created", "session.resumed"},
)
return
raw_payload = dict(payload)
raw_payload["_realtime_raw"] = True
await actor.send_json(raw_payload)
realtime_protocol.bind_sender(send_realtime_raw)
native: ServingRuntimeSessionState = self._create_runtime_session_state()
def begin_close(reason: str) -> None:
actor.closing = True
actor.close_reason = reason
if session is not None:
session.mark_closing()
event_emit_lock = asyncio.Lock()
async def send_outbound(payload: dict[str, object], *, on_accepted: Callable[[], None] | None = None) -> None:
if not attachment_ready or session is None:
await actor.send_json(dict(payload))
if on_accepted is not None:
on_accepted()
return
if realtime_protocol is None:
await send_attachment_payload(
payload,
journal=payload.get("type") not in {"session.created", "session.resumed"},
on_accepted=on_accepted,
)
return
for projected in realtime_protocol.encode_outbound_event(payload):
await send_attachment_payload(
projected,
journal=projected.get("type") not in {"session.created", "session.resumed"},
# Item/part announcements may precede the audio. Only the
# actual audio event establishes acceptance of this chunk.
on_accepted=on_accepted if projected.get("type") == "response.audio.delta" else None,
)
async def emit_event(payload: dict[str, object], *, on_accepted: Callable[[], None] | None = None) -> None:
deferred_precreate_response = False
async with event_emit_lock:
accepted, deferred_overlap_payload = await self._apply_outbound_session_event(
payload,
session=session,
actor=actor,
native=native,
realtime_protocol=realtime_protocol,
)
if not accepted:
return
await send_outbound(payload, on_accepted=on_accepted)
if deferred_overlap_payload is not None:
deferred_precreate_response = native.deferred_precreate_response
native.deferred_precreate_response = False
if deferred_overlap_payload is not None and session is not None and not actor.closing:
await start_native_append(
deferred_overlap_payload,
final=True,
precreate_response=deferred_precreate_response,
operation_id=native.committed_audio_operation_id,
retained_committed_payload=(
deferred_overlap_payload if native.committed_audio_payload is deferred_overlap_payload else None
),
)
writer_task = asyncio.create_task(actor.writer_loop(), name="duplex-session-writer")
reader_task: asyncio.Task[None] | None = None
async def read_event_loop() -> None:
nonlocal pending_turn_reservations
assert session is not None
try:
while not actor.closing:
raw = await self._receive_text(
websocket,
session.config.idle_timeout_s,
realtime_protocol=realtime_protocol,
)
if raw is None:
await actor.enqueue_event({"type": "__timeout__"})
return
if len(raw.encode("utf-8")) > _MAX_EVENT_BYTES:
await emit_event(
{"type": "error", "error": "Duplex event too large", "code": "event_too_large"}
)
continue
try:
event = json.loads(raw)
except json.JSONDecodeError:
await emit_event({"type": "error", "error": "Invalid JSON event", "code": "invalid_json"})
continue
if not isinstance(event, dict):
await emit_event(
{
"type": "error",
"error": "Duplex event must be a JSON object",
"code": "bad_event",
}
)
continue
event = normalize_duplex_input_event(event)
event_type = event.get("type")
if not isinstance(event_type, str):
await emit_event(
{"type": "error", "error": "Duplex event missing string type", "code": "bad_event"}
)
continue
if event_type in {"input.commit", "input_audio_buffer.commit"}:
if not session.reserve_pending_turn(
limit=self._duplex_session_config.max_pending_turns_per_session
):
await emit_event(
{
"type": "error",
"error": "Duplex session has too many pending input turns",
"code": "input_backpressure",
}
)
continue
event["_duplex_pending_turn_reserved"] = True
pending_turn_reservations += 1
if is_input_event(event_type) and native_response_in_progress():
event["_duplex_overlap_candidate"] = True
await actor.enqueue_event(event)
except WebSocketDisconnect:
await actor.enqueue_event({"type": "__disconnect__"})
except Exception as exc:
await emit_event({"type": "error", "error": str(exc), "code": "realtime_input_failed"})
await actor.enqueue_event({"type": "__disconnect__"})
async def next_actor_event() -> dict[str, object]:
nonlocal pending_turn_reservations, transport_detached
event = await actor.next_event()
if event.pop("_duplex_pending_turn_reserved", False) and session is not None:
session.release_pending_turn()
pending_turn_reservations = max(0, pending_turn_reservations - 1)
if (
attachment_ready
and attachment_generation is not None
and session is not None
and not await self._attachment_registry.is_current_attachment(
session.session_id,
attachment_generation,
)
):
transport_detached = True
return {"type": "__replaced_attachment__"}
return event
def native_response_in_progress() -> bool:
if session is None:
return False
if session.active_response_id is not None:
return True
if (
session.config.playback_commit_policy == DuplexPlaybackCommitPolicy.ACK_ONLY.value
and session.playback.sent_ms > session.playback.committed_ms
):
return True
if actor.active_response_task is not None and not actor.active_response_task.done():
return True
if actor.has_response_bound_append_tasks():
return True
return False
def start_chat_response() -> None:
assert session is not None
async def run_and_notify() -> None:
task = asyncio.current_task()
try:
await self._run_response(session, emit_event)
finally:
await actor.enqueue_event(
{
"type": "__chat_response_finished__",
"_response_task": task,
}
)
actor.active_response_task = asyncio.create_task(run_and_notify())
def activate_waiting_server_vad_turn() -> None:
assert session is not None
pending_turn = session.activate_server_vad_turn()
if pending_turn is not None and pending_turn.create_response:
start_chat_response()
def correlate_realtime_error(
payload: dict[str, object],
event_id: object,
) -> dict[str, object]:
if isinstance(event_id, str) and event_id:
payload["event_id"] = event_id
return payload
async def reject_and_clear_server_vad_input(
pipeline: ServerVADPipeline,
*,
code: str,
error: str,
metric_reason: str,
event_id: object,
) -> None:
assert session is not None
session.cancel_pending_input()
pipeline.reset()
self._realtime_vad_metrics.error(metric_reason)
await emit_event(
correlate_realtime_error(
{
"type": "error",
"session_id": session.session_id,
"code": code,
"error": error,
},
event_id,
)
)
await emit_event(
{
"type": "input_audio_buffer.cleared",
"session_id": session.session_id,
}
)
async def commit_server_vad_turn(
*,
item_id: str,
create_response: bool,
retain_input_bytes: int,
event_id: object = None,
) -> None:
assert session is not None
# Retain ownership until the actor consumes the terminal event so
# queued audio cannot overtake pending-turn activation.
active_response = actor.active_response_task is not None
if active_response and session.pending_server_vad_turn is not None:
released = session.discard_uncommitted_server_vad_utterance()
session.release_input_bytes(released)
self._realtime_vad_metrics.error("pending_turn_backpressure")
await emit_event(
correlate_realtime_error(
{
"type": "error",
"session_id": session.session_id,
"code": "input_backpressure",
"error": "Only one committed Server VAD turn may wait for the active response",
},
event_id,
)
)
return
if not session.stage_server_vad_audio_for_commit():
return
committed = session.commit_user_input(
append_history=not active_response,
retain_input_bytes=retain_input_bytes,
)
if committed is None:
return
if active_response:
session.stage_server_vad_turn(
committed,
create_response=create_response,
item_id=item_id,
)
else:
session.register_history_item(item_id, committed.message)
await emit_event(
self._input_committed_payload(
session,
committed,
item_id=item_id,
)
)
if create_response and not active_response:
start_chat_response()
def clear_completed_pending_silence() -> None:
task = native.pending_silence_task
if task is not None and task.done():
native.pending_silence_task = None
native.pending_silence_owner_id = None
def mark_pending_silence_superseded() -> None:
task = native.pending_silence_task
if task is None:
return
if task.done():
native.pending_silence_task = None
# Do not cancel the task or clear native_append_tail here. A
# silence append may already have reached the Engine, and local
# cancellation cannot retract that RPC. The sequencer preserves
# wire order; before_append will skip silence that has not started.
native.pending_silence_owner_id = None
def real_native_input_waiting() -> bool:
clear_completed_pending_silence()
return (
native.audio_buffer.has_pending()
or native.audio_buffer.has_reserved()
or actor.has_queued_input_events()
)
async def start_native_append(
payload: object,
*,
final: bool,
precreate_response: bool = False,
pcm_reservation: PcmAppendReservation | None = None,
operation_id: str | None = None,
retained_committed_payload: dict[str, object] | None = None,
silence_continuation: bool = False,
before_append=None,
) -> asyncio.Task[bool] | None:
if session is None:
return
if not silence_continuation:
mark_pending_silence_superseded()
append_epoch = session.epoch
append_turn_id = payload_turn_id(payload)
if append_turn_id is None:
append_turn_id = session.turn_id
request_id = self._native_stage0_request_id(session, append_epoch)
if final or precreate_response:
session.bind_request(request_id)
if precreate_response:
session.bind_response_turn(append_turn_id)
if precreate_response and session.active_response_id is None:
response_id = session.begin_response(turn_id=append_turn_id)
await emit_event(
self._response_created_payload(
session,
response_id,
epoch=append_epoch,
)
)
precreated_response_id = session.active_response_id if precreate_response else None
def _discard_retained_committed_audio() -> None:
if (
retained_committed_payload is not None
and native.committed_audio_payload is retained_committed_payload
):
session.release_input_bytes(native.clear_committed_audio())
async def _run() -> bool:
nonlocal runtime_closed
try:
session.mark_model_turn_request_started(append_turn_id, time.monotonic())
append_ok, emitted_response = await self._append_runtime_input(
session,
payload,
operation_id=(pcm_reservation.operation_id if pcm_reservation is not None else operation_id),
final=final,
send_json=emit_event,
mode="append_audio_chunk",
expected_epoch=append_epoch,
)
if append_ok:
native.native_context_locked = True
if pcm_reservation is not None:
pcm_reservation.commit()
session.release_input_bytes(pcm_reservation.byte_count)
if (
retained_committed_payload is not None
and native.committed_audio_payload is retained_committed_payload
):
session.release_input_bytes(native.clear_committed_audio())
else:
if pcm_reservation is not None:
pcm_reservation.rollback()
_discard_retained_committed_audio()
if (
not append_ok
and precreated_response_id is not None
and session.active_response_id == precreated_response_id
):
session.end_response(commit_text=False)
await emit_event(
{
"type": "response.done",
"session_id": session.session_id,
"response_id": precreated_response_id,
"epoch": session.epoch,
"committed": False,
"status": "failed",
"status_details": {
"type": "failed",
"reason": "runtime_append_failed",
},
"playback": session.playback.as_dict(),
}
)
if not append_ok and session.state == DuplexSessionState.CLOSED:
runtime_closed = True
return False
if not emitted_response and session.epoch == append_epoch:
if session.active_request_id == self._native_stage0_request_id(session, append_epoch):
session.clear_request(request_id)
if final:
await emit_event(
self._turn_controller.signal(session, DuplexTurnEventType.USER_STARTED.value)
)
return append_ok
except asyncio.CancelledError:
if pcm_reservation is not None:
pcm_reservation.rollback()
raise
except Exception as exc:
if pcm_reservation is not None:
pcm_reservation.rollback()
_discard_retained_committed_audio()
logger.exception("Native duplex append task failed: %s", exc)
await self._send_runtime_error(emit_event, "runtime_append_task_failed", exc, session=session)
if session.state != DuplexSessionState.CLOSED:
begin_close("runtime_append_task_failed")
if await self._close_runtime_session(
session,
reason="runtime_append_task_failed",
send_json=emit_event,
):
runtime_closed = True
session.close()
await emit_event(
{
"type": "session.closed",
"session_id": session.session_id,
"reason": "runtime_append_task_failed",
}
)
return False
async def _run_in_wire_order(predecessor: asyncio.Task[bool] | None) -> bool:
if predecessor is not None:
try:
predecessor_ok = await predecessor
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
raise
predecessor_ok = False
except Exception:
predecessor_ok = False
if not predecessor_ok:
if pcm_reservation is not None:
pcm_reservation.rollback()
_discard_retained_committed_audio()
return False
if actor.closing or runtime_closed or session.state != DuplexSessionState.OPEN:
if pcm_reservation is not None:
pcm_reservation.rollback()
_discard_retained_committed_audio()
return False
if before_append is not None and not before_append():
if pcm_reservation is not None:
pcm_reservation.rollback()
_discard_retained_committed_audio()
return True
if pcm_reservation is not None and not pcm_reservation.active:
_discard_retained_committed_audio()
return False
return await _run()
def _release_cancelled_retained_audio(done: asyncio.Task[bool]) -> None:
if done.cancelled():
_discard_retained_committed_audio()
return
try:
append_ok = done.result()
except Exception:
append_ok = False
if not append_ok:
_discard_retained_committed_audio()
predecessor = actor.native_append_tail
if predecessor is not None and predecessor.done():
try:
predecessor_ok = predecessor.result()
except (asyncio.CancelledError, Exception):
predecessor_ok = False
if not predecessor_ok:
# Appends already queued behind a failed predecessor keep
# their captured dependency and stop. A later wire command
# is an explicit retry and starts a new chain.
predecessor = None
task = asyncio.create_task(_run_in_wire_order(predecessor))
task.add_done_callback(_release_cancelled_retained_audio)
actor.native_append_tail = task
actor.track_append_task(
task,
epoch=append_epoch,
mode="append_audio_chunk",
final=final,
response_bound=final or precreate_response,
)
if silence_continuation:
native.pending_silence_task = task
def _clear_done_pending_silence(done: asyncio.Task[bool]) -> None:
if native.pending_silence_task is done:
native.pending_silence_task = None
native.pending_silence_owner_id = None
task.add_done_callback(_clear_done_pending_silence)
# Let this wire-order effect start before the next mailbox event can
# cancel it. Later native appends still serialize on predecessor.
await asyncio.sleep(0)
return task
async def schedule_native_silence_continuation(
payload: object,
*,
request_id: str,
owner_id: str,
response_id: str | None,
response_owned: bool,
expected_epoch: int | None,
expected_incarnation: int,
expected_model_turn_id: int | None,
send_json,
) -> bool:
del send_json
if session is None:
return False
clear_completed_pending_silence()
pending_silence = native.pending_silence_task
if pending_silence is not None and not pending_silence.done():
if pending_silence is asyncio.current_task():
return False
try:
if not await pending_silence:
return False
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
raise
return False
except Exception:
return False
clear_completed_pending_silence()
pending_silence = native.pending_silence_task
if pending_silence is not None and not pending_silence.done():
return False
append_tail = actor.native_append_tail
if (append_tail is None or append_tail.done()) and real_native_input_waiting():
return False
continuation_delay_s = max(
0.0,
float(session.capabilities.chunk_period_ms or 1000) / 1000.0,
)
if continuation_delay_s > 0:
await asyncio.sleep(continuation_delay_s)
if (
actor.native_append_tail is not append_tail
or ((append_tail is None or append_tail.done()) and real_native_input_waiting())
or self._native_silence_continuation_is_stale(
session,
request_id=request_id,
response_id=response_id,
response_owned=response_owned,
expected_epoch=expected_epoch,
expected_incarnation=expected_incarnation,
expected_model_turn_id=expected_model_turn_id,
)
):
return False
def _still_valid() -> bool:
return not real_native_input_waiting() and not self._native_silence_continuation_is_stale(
session,
request_id=request_id,
response_id=response_id,
response_owned=response_owned,
expected_epoch=expected_epoch,
expected_incarnation=expected_incarnation,
expected_model_turn_id=expected_model_turn_id,
)
native.pending_silence_owner_id = owner_id
task = await start_native_append(
payload,
final=False,
silence_continuation=True,
before_append=_still_valid,
)
if task is None:
native.pending_silence_owner_id = None
return False
return True
async def wait_for_native_append_tail() -> bool:
predecessor = actor.native_append_tail
if predecessor is None:
return True
try:
return await predecessor
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
raise
return False
except Exception:
# The append task already emitted its runtime error. Preserve
# wire order before the update path decides whether to continue.
return False
async def start_runtime_append(
payload: object,
*,
final: bool,
mode: str = "append_tokens",
) -> None:
"""Schedule non-native runtime appends without blocking WS input.
Model-native appends use ``start_native_append`` because they can
create data-plane response streams. Generic runtime appends still
need the same control-plane isolation so cancel/close can preempt a
slow engine append.
"""
if session is None:
return
append_epoch = session.epoch
async def _run() -> None:
nonlocal runtime_closed
try:
append_ok, emitted_response = await self._append_runtime_input(
session,
payload,
final=final,
send_json=emit_event,
mode=mode,
expected_epoch=append_epoch,
)
if not append_ok:
if session.state == DuplexSessionState.CLOSED:
runtime_closed = True
return
if not emitted_response and session.epoch == append_epoch:
await emit_event(self._turn_controller.signal(session, DuplexTurnEventType.USER_STARTED.value))
except asyncio.CancelledError:
raise
except Exception as exc:
logger.exception("Duplex runtime append task failed: %s", exc)
await self._send_runtime_error(emit_event, "runtime_append_task_failed", exc, session=session)
task = asyncio.create_task(_run())
actor.track_append_task(
task,
epoch=append_epoch,
mode=mode,
final=final,
response_bound=final,
)
try:
handshake = await self._open_session(
websocket,
emit_event,
realtime_protocol=realtime_protocol,
attachment_send=attachment_send,
attachment_close=attachment_close,
)
if handshake is None:
return
session = handshake.session
if initial_update_error := self._runtime_session_candidate_update_error(
session, session.config, realtime_protocol=realtime_protocol
):
await emit_event(
correlate_realtime_error(
initial_update_error,
handshake.event_id,
)
)
return
if handshake.resumed:
native = self._serving_session_states[session.session_id]
actor.tasks = self._session_tasks[session.session_id]
persisted_protocol = self._realtime_protocols.get(session.session_id)
if persisted_protocol is None:
raise RuntimeError(f"Missing Realtime protocol state for resumed session {session.session_id}")
realtime_protocol = persisted_protocol
realtime_protocol.bind_sender(send_realtime_raw)
attachment_generation = handshake.attachment_generation
attachment_ready = True
resume_credential_delivered = True
runtime_opened = True
self._ensure_lifecycle_listener()
reader_task = asyncio.create_task(read_event_loop(), name="duplex-session-reader")
else:
self._serving_session_states[session.session_id] = native
self._session_tasks[session.session_id] = actor.tasks
if realtime_protocol is not None:
self._realtime_protocols[session.session_id] = realtime_protocol
native.silence_continuation_scheduler = schedule_native_silence_continuation
if realtime_protocol is not None:
realtime_protocol.bind_native_input_append(self._uses_native_input_append(session))
session.config.playback_commit_policy = DuplexPlaybackCommitPolicy.ACK_ONLY.value
if not handshake.resumed:
open_result = await self._open_runtime_session(session, emit_event)
if open_result is False:
return
runtime_opened = True
try:
await self._configure_server_vad(session)
except Exception as exc:
logger.exception("Server VAD initialization failed: %s", exc)
self._realtime_vad_metrics.error("initialization")
await emit_event(
correlate_realtime_error(
{
"type": "error",
"session_id": session.session_id,
"code": "server_vad_initialization_failed",
"error": str(exc),
},
handshake.event_id,
)
)
return
created_attachment = await self._attachment_registry.create(
session.session_id,
incarnation=session.incarnation,
send=attachment_send,
close=attachment_close,
)
attachment_generation = created_attachment.attachment_generation
attachment_ready = True
self._lease_generations[session.session_id] = 0
self._ensure_lifecycle_listener()
created_payload: dict[str, object] = {
"type": "session.created",
"session": session.as_public_dict(),
}
if session.capabilities.supports_session_resume:
created_payload.update(
{
"incarnation": session.incarnation,
"attachment_generation": attachment_generation,
"resume_token": created_attachment.resume_token.plaintext,
}
)
if isinstance(open_result, dict):
created_payload["runtime_control"] = self._redact_runtime_control_result(open_result)
if realtime_protocol is not None:
realtime_protocol.commit_realtime_turn_detection_update(session.config.server_vad)
await emit_event(created_payload)
resume_credential_delivered = session.capabilities.supports_session_resume
reader_task = asyncio.create_task(read_event_loop(), name="duplex-session-reader")
while True:
event = await next_actor_event()
event_type = event.get("type")
if event_type == "__replaced_attachment__":
return
if event_type == "__chat_response_finished__":
completed_task = event.get("_response_task")
if completed_task is not actor.active_response_task:
continue
actor.active_response_task = None
activate_waiting_server_vad_turn()
continue
if event_type == "__timeout__":
begin_close("timeout")
native.audio_buffer.clear()
session.release_all_input_bytes()
native.input_since_commit = False
native.speech_since_commit = False
native.clear_committed_audio()
await actor.cancel_append_tasks()
await self._cancel_native_data_plane_stream(session)
await self._cancel_active_response(
session,
actor.active_response_task,
emit_event,
reason="timeout",
notify=True,
)
actor.active_response_task = None
runtime_closed = await self._close_runtime_session(
session,
reason="timeout",
send_json=emit_event,
)
if not runtime_closed:
return
session.close()
await emit_event(
{
"type": "session.closed",
"session_id": session.session_id,
"reason": "timeout",
}
)
return
if event_type == "__disconnect__":
transport_detached = True
return
if not isinstance(event_type, str):
continue
if event_type == "session.heartbeat":
touch_session = getattr(self._chat_service.engine_client, "touch_duplex_session_async", None)
if not callable(touch_session):
await emit_event(
{
"type": "error",
"error": "Duplex runtime does not expose heartbeat control",
"code": "runtime_touch_unsupported",
}
)
continue
try:
await touch_session(
session.session_id,
fence=DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=session.turn_id,
incarnation=session.incarnation,
),
activity=DuplexLeaseActivity.HEARTBEAT,
)
except Exception as exc:
await emit_event(
{
"type": "error",
"error": str(exc),
"code": "runtime_touch_failed",
}
)
continue
await emit_event(
{
"type": "session.heartbeat_ack",
"session_id": session.session_id,
}
)
continue
if event_type == "session.event_ack":
acknowledged = event.get("server_event_seq")
if not isinstance(acknowledged, int) or acknowledged < 0:
await emit_event(
{
"type": "error",
"error": "session.event_ack requires a non-negative server_event_seq",
"code": "invalid_event_ack",
}
)
continue
try:
await self._attachment_registry.acknowledge(session.session_id, acknowledged)
except ValueError as exc:
await emit_event(
{
"type": "error",
"error": str(exc),
"code": "invalid_event_ack",
}
)
continue
if event_type == "turn.signal" and event.get("event") in {
"input.cancel",
"response.cancel",
"barge_in",
}:
payload = event.get("payload")
normalized_event = dict(payload) if isinstance(payload, dict) else {}
normalized_event.update(event)
normalized_event["type"] = event["event"]
event = normalized_event
event_type = str(event["type"])
if event_type == "session.close":
with suppress(asyncio.TimeoutError):
await asyncio.wait_for(actor.output_queue.join(), timeout=1.0)
begin_close("session_close")
if session.state == DuplexSessionState.CLOSED:
runtime_closed = True
return
native.audio_buffer.clear()
session.release_all_input_bytes()
native.input_since_commit = False
native.speech_since_commit = False
native.clear_committed_audio()
await actor.cancel_append_tasks()
await self._cancel_native_data_plane_stream(session)
await self._cancel_active_response(
session,
actor.active_response_task,
emit_event,
reason="session_close",
)
runtime_closed = await self._close_runtime_session(
session,
reason="session_close",
send_json=emit_event,
)
actor.active_response_task = None
if not runtime_closed:
return
await emit_event({"type": "session.closed", "session_id": session.session_id})
session.close()
return
if event_type == "input_audio_buffer.clear":
native.audio_buffer.clear()
session.release_all_input_bytes()
server_vad_pipeline = self._server_vad_pipelines.get(session.session_id)
if server_vad_pipeline is not None:
server_vad_pipeline.reset()
native.input_since_commit = False
native.speech_since_commit = False
native.clear_committed_audio()
cancelled = session.cancel_pending_input()
await emit_event(
{
"type": "input_audio_buffer.cleared",
"session_id": session.session_id,
"epoch": session.epoch,
"drained_input_events": 0,
"cancelled": cancelled,
}
)
continue
if event_type == "barge_in" and not session.capabilities.supports_barge_in:
await emit_event(self._barge_in_unsupported_error(session))
continue
if event_type in {"input.cancel", "response.cancel", "barge_in", "output_audio_buffer.clear"}:
cancel_reason = (
"output_audio_buffer_clear"
if event_type == "output_audio_buffer.clear"
else "client_cancelled"
if event_type == "response.cancel"
else "barge_in"
)
cancelled_fence = DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=session.turn_id,
incarnation=session.incarnation,
)
if event_type == "response.cancel":
requested_response_id = event.get("response_id")
has_active_response_work = native_response_in_progress()
if isinstance(requested_response_id, str) and (
session.active_response_id is None or requested_response_id != session.active_response_id
):
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"code": "response_not_active",
"error": f"Response is not active: {requested_response_id}",
}
)
continue
if event_type == "response.cancel":
if not has_active_response_work:
if realtime_protocol is not None and isinstance(requested_response_id, str):
continue
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"code": "response_not_active",
"error": "response.cancel requires an active response",
}
)
continue
had_native_unbuffered_append = (
self._uses_native_input_append(session)
and native.input_since_commit
and not native.audio_buffer.has_pending()
)
playback_was_active = self._assistant_playback_active(session)
if event_type in {"input.cancel", "barge_in"}:
native.audio_buffer.clear()
session.release_all_input_bytes()
native.input_since_commit = False
native.speech_since_commit = False
native.clear_committed_audio()
had_native_append = await actor.cancel_append_tasks(
response_bound_only=event_type in {"response.cancel", "output_audio_buffer.clear"},
)
if event_type == "response.cancel":
session.release_input_bytes(native.clear_committed_audio())
had_native_stream = native.data_plane_task is not None
cancelled = await self._cancel_active_response(
session,
actor.active_response_task,
emit_event,
reason=cancel_reason,
)
had_native_stream = await self._cancel_native_data_plane_stream(session) or had_native_stream
if not cancelled and (had_native_stream or had_native_append or had_native_unbuffered_append):
old_epoch = session.epoch
old_response_id = session.active_response_id
committed_ms = session.playback.committed_ms
self._commit_played_response_history(session, old_response_id, committed_ms)
new_epoch, old_playback = self._advance_barge_in_epoch(session)
await emit_event(
{
"type": "audio.cancelled",
"session_id": session.session_id,
"response_id": old_response_id,
"reason": cancel_reason,
"cancelled_epoch": old_epoch,
"epoch": new_epoch,
"committed_ms": committed_ms,
"playback": old_playback,
}
)
cancelled = True
if not cancelled and playback_was_active:
old_epoch = session.epoch
committed_ms = session.playback.committed_ms
self._commit_played_response_history(session, session.last_response_id, committed_ms)
new_epoch, old_playback = self._advance_barge_in_epoch(session)
await emit_event(
{
"type": "audio.cancelled",
"session_id": session.session_id,
"response_id": session.last_response_id,
"reason": cancel_reason,
"cancelled_epoch": old_epoch,
"epoch": new_epoch,
"committed_ms": committed_ms,
"playback": old_playback,
}
)
cancelled = True
if not cancelled and event_type == "response.cancel":
old_epoch = session.epoch
old_response_id = session.active_response_id
committed_ms = session.playback.committed_ms
self._commit_played_response_history(session, old_response_id, committed_ms)
new_epoch, old_playback = self._advance_barge_in_epoch(session)
await emit_event(
{
"type": "audio.cancelled",
"session_id": session.session_id,
"response_id": old_response_id,
"reason": cancel_reason,
"cancelled_epoch": old_epoch,
"epoch": new_epoch,
"committed_ms": committed_ms,
"playback": old_playback,
}
)
cancelled = True
if not cancelled and event_type == "output_audio_buffer.clear":
old_playback = session.playback.as_dict()
committed_ms = session.playback.committed_ms
session.clear_playback_cursor()
await emit_event(
{
"type": "audio.cancelled",
"session_id": session.session_id,
"response_id": session.active_response_id,
"reason": cancel_reason,
"cancelled_epoch": session.epoch,
"epoch": session.epoch,
"committed_ms": committed_ms,
"playback": old_playback,
}
)
actor.active_response_task = None
activate_waiting_server_vad_turn()
continue
if not cancelled:
await self._cancel_pending_input(session, emit_event, reason="barge_in")
if not await self._signal_runtime_session(
session,
"barge_in",
emit_event,
fence=cancelled_fence,
next_fence=(
DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=session.turn_id,
incarnation=session.incarnation,
)
if session.epoch > cancelled_fence.epoch
else None
),
):
continue
actor.active_response_task = None
activate_waiting_server_vad_turn()
continue
if event_type == "turn.signal":
turn_event = event.get("event")
if isinstance(turn_event, str):
if turn_event == "barge_in" and not session.capabilities.supports_barge_in:
await emit_event(self._barge_in_unsupported_error(session))
continue
if turn_event == "session.update":
realtime_event_id = event.get("realtime_event_id")
async def emit_update_event(update_event: dict[str, object]) -> None:
if isinstance(realtime_event_id, str) and realtime_event_id:
update_event = {**update_event, "realtime_event_id": realtime_event_id}
await emit_event(update_event)
def reject_update() -> None:
if realtime_protocol is not None:
realtime_protocol.reject_realtime_turn_detection_update()
payload = event.get("payload")
if not isinstance(payload, dict):
await emit_update_event(
{
"type": "error",
"session_id": session.session_id,
"code": "bad_event",
"error": "session.update requires a session payload",
}
)
reject_update()
continue
if not await wait_for_native_append_tail():
await emit_update_event(
{
"type": "error",
"code": "session_update_aborted",
"error": "session.update was not applied because the preceding append failed",
}
)
reject_update()
continue
if (
realtime_protocol is not None
and not self._uses_native_input_append(session)
and session.config.server_vad is not None
and session.turn_detection_config_locked
and realtime_protocol.pending_realtime_input_audio_config_changed()
):
await emit_update_event(
{
"type": "error",
"session_id": session.session_id,
"code": "unsupported_audio_format",
"error": (
"session.update cannot change Server VAD input format or sample rate "
"after the first audio append"
),
"param": "audio.input.format",
}
)
reject_update()
continue
runtime_update_error = self._runtime_session_update_error(session, payload)
if runtime_update_error is not None:
await emit_update_event(runtime_update_error)
reject_update()
continue
previous_config = session.config
candidate_config = deepcopy(previous_config)
session.replace_config(candidate_config)
try:
update_error = self._apply_session_update(session, payload)
finally:
session.replace_config(previous_config)
server_vad_changed = candidate_config.server_vad != previous_config.server_vad
if update_error is not None:
await emit_update_event(update_error)
reject_update()
continue
runtime_update_error = self._runtime_session_candidate_update_error(
session,
candidate_config,
realtime_protocol=realtime_protocol,
)
if runtime_update_error is not None:
await emit_update_event(runtime_update_error)
reject_update()
continue
try:
candidate_runtime_config = self._runtime_config_for_session_update(
session,
candidate_config,
)
except ServingRuntimeConfigError as exc:
await emit_update_event(
{
"type": "error",
"session_id": session.session_id,
"code": exc.code,
"error": str(exc),
}
)
reject_update()
continue
candidate_server_vad_pipeline = None
if server_vad_changed:
try:
candidate_server_vad_pipeline = await self._prepare_server_vad_pipeline(
session,
candidate_config,
)
except Exception as exc:
logger.exception("Server VAD session update failed: %s", exc)
self._realtime_vad_metrics.error("initialization")
await emit_update_event(
{
"type": "error",
"session_id": session.session_id,
"code": "server_vad_initialization_failed",
"error": str(exc),
}
)
reject_update()
continue
runtime_updated = await self._signal_runtime_session(
session,
turn_event,
emit_update_event,
session_config=candidate_config.as_dict(),
runtime_config=candidate_runtime_config,
)
if not runtime_updated:
reject_update()
continue
session.replace_config(candidate_config)
session.replace_runtime_config(candidate_runtime_config)
if server_vad_changed:
self._install_server_vad_pipeline(
session,
candidate_server_vad_pipeline,
)
if realtime_protocol is not None:
realtime_protocol.bind_native_input_append(self._uses_native_input_append(session))
realtime_protocol.commit_realtime_turn_detection_update(session.config.server_vad)
await emit_update_event(
{
"type": "session.updated",
"session": session.as_public_dict(),
}
)
continue
if turn_event == "conversation.item.create":
payload = event.get("payload")
item = payload.get("item") if isinstance(payload, dict) else None
item_type = item.get("type") if isinstance(item, dict) else None
prepare_function_output = getattr(
self._serving_runtime_adapter,
"runtime_config_for_function_output",
None,
)
if item_type == "function_call_output" and callable(prepare_function_output):
if not await wait_for_native_append_tail():
continue
try:
candidate_runtime_config = prepare_function_output(
item,
dict(session.runtime_config),
)
except ServingRuntimeConfigError as exc:
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"code": exc.code,
"error": str(exc),
}
)
continue
if not await self._signal_runtime_session(
session,
"session.update",
emit_event,
runtime_config=candidate_runtime_config,
):
continue
session.replace_runtime_config(candidate_runtime_config)
await emit_event(
{
"type": "conversation.item.created",
"session_id": session.session_id,
"item": item,
"created": True,
}
)
await self._maybe_continue_native_response(
emit_event,
session=session,
expected_epoch=session.epoch,
expected_model_turn_id=session.turn_id,
)
continue
history_item = payload.get("history_item") if isinstance(payload, dict) else None
if not isinstance(history_item, dict):
history_item = item
message = self._realtime_item_to_history_message(history_item)
item_id = item.get("id") if isinstance(item, dict) else None
if message is not None:
session.append_history_message(message)
session.register_history_item(item_id if isinstance(item_id, str) else None, message)
await emit_event(
{
"type": "conversation.item.created",
"session_id": session.session_id,
"item": item,
"created": message is not None,
}
)
continue
if turn_event == "conversation.item.delete":
payload = event.get("payload")
item_id = payload.get("item_id") if isinstance(payload, dict) else None
deleted = session.delete_history_item(item_id) if isinstance(item_id, str) else False
await emit_event(
{
"type": "conversation.item.deleted",
"session_id": session.session_id,
"item_id": item_id,
"deleted": deleted,
}
)
continue
if turn_event == "conversation.item.truncate":
payload = event.get("payload")
item_id = payload.get("item_id") if isinstance(payload, dict) else None
audio_end_ms = payload.get("audio_end_ms") if isinstance(payload, dict) else None
truncated = (
session.truncate_history_item(
item_id,
audio_end_ms=int(audio_end_ms) if isinstance(audio_end_ms, int | float) else 0,
hard=True,
)
if isinstance(item_id, str)
else False
)
await emit_event(
{
"type": "conversation.item.truncated",
"session_id": session.session_id,
"item_id": item_id,
"content_index": (
payload.get("content_index", 0) if isinstance(payload, dict) else 0
),
"audio_end_ms": audio_end_ms,
"truncated": truncated,
}
)
continue
await emit_event(self._turn_controller.signal(session, turn_event, event))
else:
await emit_event({"type": "error", "error": "turn.signal requires event", "code": "bad_event"})
continue
if event_type == "playback.ack":
await self._handle_playback_ack(session, event, emit_event)
continue
if event_type == "input.text.append":
session.mark_user_input_activity()
text = event.get("text")
if not isinstance(text, str):
await emit_event(
{
"type": "error",
"error": "input.text.append requires text",
"code": "bad_event",
}
)
continue
if self._uses_native_input_append(session):
await emit_event(
{
"type": "error",
"error": "The selected native duplex runtime accepts audio append only",
"code": "native_text_append_unsupported",
}
)
continue
else:
session.append_text(text)
if session.capabilities.supports_input_append:
await start_runtime_append(text, final=False, mode="append_tokens")
continue
if event_type == "input_audio_buffer.append":
session.mark_user_input_activity()
audio = event.get("audio") or event.get("data")
if not isinstance(audio, str):
await emit_event(
{
"type": "error",
"error": "input.audio.append requires audio",
"code": "bad_event",
}
)
continue
if not session.capabilities.supports_barge_in and self._event_requests_barge_in(event):
await emit_event(self._barge_in_unsupported_error(session))
event = dict(event)
event.pop("force_barge_in", None)
for key in ("overlap_action", "overlap"):
value = event.get(key)
if isinstance(value, str) and value.strip().lower() in {
"barge_in",
"interrupt",
"cancel",
}:
event.pop(key, None)
fmt = event.get("format") if isinstance(event.get("format"), str) else "pcm16"
default_sample_rate_hz = 16000
sr_raw = event.get("sample_rate_hz") or event.get("sample_rate")
sample_rate_hz = sr_raw if isinstance(sr_raw, int | float) else default_sample_rate_hz
native_input = self._uses_native_input_append(session)
server_vad_config = session.config.server_vad
turn_based_server_vad = server_vad_config is not None and not native_input
if not turn_based_server_vad:
try:
audio, fmt, sample_rate_hz = convert_input_audio_with_rate(
audio,
fmt,
sample_rate_hz=sample_rate_hz,
)
except ValueError as exc:
await emit_event({"type": "error", "error": str(exc), "code": "bad_event"})
continue
if isinstance(fmt, str) and fmt.lower() in {"pcm16", "pcm_s16le", "s16le"}:
await emit_event(
{
"type": "error",
"error": "input_audio_buffer.append pcm16 audio could not be decoded",
"code": "bad_audio",
}
)
continue
if server_vad_config is None:
session.lock_turn_detection_config()
force_listen = bool(event.get("force_listen", False))
payload = {
"type": "audio",
"audio": audio,
"format": fmt,
"sample_rate_hz": sample_rate_hz,
"force_listen": force_listen,
}
video_frames = event.get("video_frames")
if isinstance(video_frames, list):
frames = [frame for frame in video_frames if isinstance(frame, str) and frame]
if frames:
payload["video_frames"] = frames
if server_vad_config is not None:
realtime_event_id = event.get("_realtime_event_id")
pipeline = self._server_vad_pipelines.get(session.session_id)
if pipeline is None:
await emit_event(
correlate_realtime_error(
{
"type": "error",
"session_id": session.session_id,
"code": "internal_error",
"error": "server_vad is configured but its detector pipeline is unavailable",
},
realtime_event_id,
)
)
continue
speech_was_active = pipeline.speech_active
reserved_bytes = 0
previous_scratch_bytes = pipeline.scratch_bytes
try:
if native_input:
if fmt != "pcm_f32le" or sample_rate_hz != pipeline.sample_rate_hz:
raise ValueError("native server_vad input must be mono PCM float32 at 16000 Hz")
raw_audio = base64.b64decode(audio, validate=True)
if len(raw_audio) % np.dtype("<f4").itemsize:
raise ValueError("native server_vad input contains an incomplete float32 sample")
normalized_audio = np.frombuffer(raw_audio, dtype="<f4")
accepted_audio = normalized_audio.size > 0
else:
if fmt != "pcm16":
raise ValueError("server_vad input must be mono PCM16")
source_sample_rate_hz = validate_input_sample_rate_hz(sample_rate_hz)
if source_sample_rate_hz not in {16_000, 24_000}:
raise ValueError("server_vad PCM16 input sample rate must be 16000 or 24000 Hz")
if (
pipeline.source_sample_rate_hz is not None
and source_sample_rate_hz != pipeline.source_sample_rate_hz
):
raise ValueError(
"server_vad input sample rate cannot change within a continuous audio stream"
)
try:
raw_audio = base64.b64decode(audio, validate=True)
except (binascii.Error, ValueError) as exc:
raise ValueError(
"server_vad input audio must be valid base64-encoded PCM16"
) from exc
if len(raw_audio) % np.dtype("<i2").itemsize:
raise ValueError("server_vad PCM16 input contains an incomplete sample")
source_samples = len(raw_audio) // np.dtype("<i2").itemsize
accepted_audio = source_samples > 0
except (binascii.Error, ValueError) as exc:
await emit_event(
correlate_realtime_error(
{
"type": "error",
"session_id": session.session_id,
"code": "bad_audio",
"error": str(exc),
"param": "audio",
},
realtime_event_id,
)
)
continue
if not native_input:
normalized_sample_upper_bound = (
source_samples
if source_sample_rate_hz == pipeline.sample_rate_hz
else ((source_samples + 2) // 3) * 2
)
reserved_bytes = normalized_sample_upper_bound * np.dtype(np.float32).itemsize
if not session.reserve_input_bytes(
reserved_bytes,
limit=self._duplex_session_config.max_pending_input_bytes_per_session,
):
await reject_and_clear_server_vad_input(
pipeline,
code="input_backpressure",
error="Server VAD input exceeds the per-session input limit",
metric_reason="input_backpressure",
event_id=realtime_event_id,
)
continue
try:
vad_batch = (
await pipeline.push(normalized_audio)
if native_input
else await pipeline.push_pcm16(
raw_audio,
source_sample_rate_hz=source_sample_rate_hz,
)
)
except Exception as exc:
logger.exception("Server VAD inference failed: %s", exc)
await reject_and_clear_server_vad_input(
pipeline,
code="server_vad_inference_failed",
error=str(exc),
metric_reason="inference",
event_id=realtime_event_id,
)
continue
if not native_input:
retained_delta = (
sum(frame.samples.nbytes for frame in vad_batch.frames)
+ pipeline.scratch_bytes
- previous_scratch_bytes
)
session.release_input_bytes(max(0, reserved_bytes - retained_delta))
self._realtime_vad_metrics.observe_inference(vad_batch.inference_ms)
prefix_samples = round(server_vad_config.prefix_padding_ms * pipeline.sample_rate_hz / 1000)
speech_started = False
speech_stopped = False
for frame_index, vad_frame in enumerate(vad_batch.frames):
decision = vad_frame.decision
if not native_input:
released_bytes = session.append_server_vad_frame(
vad_frame.samples,
speech_started=decision.speech_started,
speech_stopped=decision.speech_stopped,
prefix_samples=prefix_samples,
)
session.release_input_bytes(released_bytes)
if decision.speech_started:
speech_started = True
item_id = f"item_{uuid.uuid4().hex}"
session.begin_server_vad_speech(item_id)
await emit_event(
{
"type": "input_audio_buffer.speech_started",
"session_id": session.session_id,
"item_id": item_id,
"audio_start_ms": decision.audio_start_ms or 0,
}
)
if decision.speech_stopped:
speech_stopped = True
item_id = session.finish_server_vad_speech() or f"item_{uuid.uuid4().hex}"
await emit_event(
{
"type": "input_audio_buffer.speech_stopped",
"session_id": session.session_id,
"item_id": item_id,
"audio_end_ms": decision.audio_end_ms or 0,
}
)
if decision.endpoint_delay_ms is not None:
self._realtime_vad_metrics.observe_endpoint_delay(decision.endpoint_delay_ms)
if not native_input:
remaining_frame_bytes = sum(
frame.samples.nbytes for frame in vad_batch.frames[frame_index + 1 :]
)
await commit_server_vad_turn(
item_id=item_id,
create_response=server_vad_config.create_response,
retain_input_bytes=remaining_frame_bytes + pipeline.scratch_bytes,
event_id=realtime_event_id,
)
if accepted_audio:
session.lock_turn_detection_config()
if not native_input:
continue
speech_active = pipeline.speech_active
contained_speech = speech_was_active or speech_started or speech_stopped or speech_active
payload["is_speech"] = contained_speech
payload["vad"] = {
"backend": "silero",
"is_speech": contained_speech,
"speech_active": speech_active,
"speech_started": speech_started,
"speech_stopped": speech_stopped,
}
if speech_active:
payload["force_listen"] = True
# Speech/silence tag for the Stage0 turn-ended latch.
payload["is_speech"] = self._input_looks_like_speech(event, payload, session=session)
defer_native_append = False
buffer_overlap_audio = True
if native_input:
mark_pending_silence_superseded()
overlap_active = native_response_in_progress() and (
not self._session_auto_responds(session)
or (
session.capabilities.supports_barge_in
and (
session.config.overlap_policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value
or self._event_requests_barge_in(event)
)
)
)
if overlap_active:
decision = self._overlap_decision(session, event, payload)
await self._emit_overlap_decision(emit_event, session, decision)
action = decision.get("action")
if action == "drop":
if realtime_protocol is not None:
await realtime_protocol.discard_pending_input_audio(
audio_end_ms=self._input_audio_duration_ms(event, payload)
)
continue
if action == "listen":
buffer_overlap_audio = bool(decision.get("buffer_audio", True))
defer_native_append = bool(decision.get("defer_runtime_append", True))
if (
not buffer_overlap_audio
and realtime_protocol is not None
and decision.get("preserve_realtime_input") is not True
):
await realtime_protocol.discard_pending_input_audio(
audio_end_ms=self._input_audio_duration_ms(event, payload)
)
if decision.get("force_listen", True) is True:
payload["force_listen"] = True
else:
event["force_barge_in"] = True
cancelled_fence = DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=session.turn_id,
incarnation=session.incarnation,
)
playback_was_active = self._assistant_playback_active(session)
buffer_overlap_audio = True
defer_native_append = False
native.audio_buffer.clear_force_listen()
session.reset_overlap_speech()
native.input_since_commit = False
native.speech_since_commit = False
await actor.cancel_append_tasks()
had_native_stream = native.data_plane_task is not None
cancel_reason = str(decision.get("cancel_reason") or "barge_in")
cancelled = await self._cancel_active_response(
session,
actor.active_response_task,
emit_event,
reason=cancel_reason,
)
had_native_stream = (
await self._cancel_native_data_plane_stream(session) or had_native_stream
)
if not cancelled and had_native_stream:
old_epoch = session.epoch
old_response_id = session.active_response_id
committed_ms = session.playback.committed_ms
self._commit_played_response_history(session, old_response_id, committed_ms)
new_epoch, old_playback = self._advance_barge_in_epoch(session)
await emit_event(
{
"type": "audio.cancelled",
"session_id": session.session_id,
"response_id": old_response_id,
"reason": cancel_reason,
"cancelled_epoch": old_epoch,
"epoch": new_epoch,
"committed_ms": committed_ms,
"playback": old_playback,
}
)
cancelled = True
if not cancelled and playback_was_active:
old_epoch = session.epoch
committed_ms = session.playback.committed_ms
self._commit_played_response_history(
session, session.last_response_id, committed_ms
)
new_epoch, old_playback = self._advance_barge_in_epoch(session)
await emit_event(
{
"type": "audio.cancelled",
"session_id": session.session_id,
"response_id": session.last_response_id,
"reason": cancel_reason,
"cancelled_epoch": old_epoch,
"epoch": new_epoch,
"committed_ms": committed_ms,
"playback": old_playback,
}
)
cancelled = True
if session.epoch > cancelled_fence.epoch:
if not await self._signal_runtime_session(
session,
"barge_in",
emit_event,
fence=cancelled_fence,
next_fence=DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=session.turn_id,
incarnation=session.incarnation,
),
):
continue
actor.active_response_task = None
elif not self._session_auto_responds(session) and not self._input_looks_like_speech(
event, payload, session=session
):
# Turn-mode only: skip silent chunks so they don't open a
# response. In auto-respond (full-duplex) mode the model owns
# the speak/listen decision and MUST receive silence units --
# the official model typically starts speaking during the
# silence after a question.
# No response_id: a silence-skip listen can arrive
# while a response is still streaming and must not
# read as that response's terminal decision.
await emit_event(
{
"type": "response.listen",
"session_id": session.session_id,
"epoch": session.epoch,
"reason": "silence_or_noise",
}
)
continue
if self._should_force_listen_for_auto_response_overlap(session, event, payload):
# Auto-response keeps a long-lived native Stage0 stream.
# While assistant audio is still active, silence from the
# browser should advance the model in listen mode rather
# than letting the same stream start another assistant
# segment. Speech/barge-in chunks remain model-owned.
payload["force_listen"] = True
if not buffer_overlap_audio:
continue
session.mark_user_input_activity()
native.input_since_commit = True
native.speech_since_commit = native.speech_since_commit or self._input_looks_like_speech(
event, payload, session=session
)
try:
raw_audio_bytes = self._native_audio_payload_size_bytes(payload)
if not session.reserve_input_bytes(
raw_audio_bytes,
limit=self._duplex_session_config.max_pending_input_bytes_per_session,
):
await emit_event(
{
"type": "error",
"error": "Duplex session pending input exceeds server limit",
"code": "input_backpressure",
}
)
continue
allow_emit = not defer_native_append and (
realtime_protocol is None
or event_type != "input_audio_buffer.append"
# Full-duplex: emit each ~chunk_period of audio so the model
# runs per-chunk generation (speak/listen) without an explicit
# response.create, matching the official duplex_generate loop.
or self._session_auto_responds(session)
)
pcm_reservation = native.audio_buffer.prepare_append(
payload,
operation_id=uuid.uuid4().hex,
chunk_period_ms=session.capabilities.chunk_period_ms or 1000,
allow_emit=allow_emit,
)
except ValueError as exc:
session.release_input_bytes(raw_audio_bytes)
await emit_event({"type": "error", "error": str(exc), "code": "bad_event"})
continue
if pcm_reservation is None:
continue
if pcm_reservation.byte_count == 0:
session.release_input_bytes(raw_audio_bytes)
payload = pcm_reservation.payload
else:
session.append_audio(audio, fmt=fmt, sample_rate_hz=sample_rate_hz)
if native_input:
await start_native_append(
payload,
final=False,
pcm_reservation=pcm_reservation,
)
continue
if session.capabilities.supports_input_append:
await start_runtime_append(payload, final=False, mode="append_audio_chunk")
continue
if event_type in {"input.commit", "input_audio_buffer.commit", "response.create"}:
if (
event_type == "response.create"
and session.config.server_vad is not None
and actor.active_response_task is not None
):
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"code": "response_already_active",
"error": "response.create must wait for the active response to complete",
}
)
continue
item_id = event.get("item_id")
realtime_validated_audio_commit = (
event_type == "input_audio_buffer.commit" and isinstance(item_id, str) and bool(item_id)
)
if (
self._uses_native_input_append(session)
and event_type in {"input.commit", "input_audio_buffer.commit"}
and not await wait_for_native_append_tail()
):
continue
if event_type == "input_audio_buffer.commit" and event.get("is_speech") is False:
native.input_since_commit = False
native.speech_since_commit = False
native.audio_buffer.clear()
session.release_all_input_bytes()
native.clear_committed_audio()
await emit_event(
{
"type": "input.committed",
"session_id": session.session_id,
"item_id": item_id,
"turn_id": session.turn_id,
"epoch": session.epoch,
"empty": True,
"is_speech": False,
"no_response": True,
}
)
# No response_id: this silence listen answers an empty
# commit, it is not a response terminal.
await emit_event(
{
"type": "response.listen",
"session_id": session.session_id,
"epoch": session.epoch,
"reason": "silence_or_noise",
}
)
continue
should_create_response = (
event_type == "response.create"
or bool(event.get("response_create", event_type == "input.commit"))
or (event_type == "input_audio_buffer.commit" and self._session_auto_responds(session))
)
precreate_response_requested = event_type == "response.create" or bool(
event.get("response_create", event_type == "input.commit")
)
if event_type == "response.create":
response_payload = event.get("response")
if isinstance(response_payload, dict):
response_options_error = self._apply_response_create_options(session, response_payload)
if response_options_error is not None:
error_message = (
"The selected native duplex runtime does not support generation "
"overrides for instructions, voice, temperature, max tokens, tools, or "
"tool_choice."
if response_options_error == "unsupported_native_response_options"
else "response.create cannot reserve options while another response is active."
)
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"epoch": session.epoch,
"code": response_options_error,
"error": error_message,
}
)
continue
if self._uses_native_input_append(session) and event_type == "input_audio_buffer.commit":
has_pending_native_audio = (
native.input_since_commit
or native.audio_buffer.has_pending()
or native.committed_audio_payload is not None
or realtime_validated_audio_commit
)
if (
not has_pending_native_audio
and not actor.native_append_tasks
and native.data_plane_task is None
):
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"epoch": session.epoch,
"code": "input_audio_buffer_empty",
"error": "input_audio_buffer.commit requires a non-empty input audio buffer.",
}
)
continue
commit_action = decide_commit_action(
CommitSnapshot(
auto_responds=self._session_auto_responds(session),
speech_since_commit=native.speech_since_commit,
active_response_id=session.active_response_id,
overlap_speech_ms=session.overlap_speech_ms,
native_response_in_progress=native_response_in_progress(),
playback_active=self._assistant_playback_active(session),
)
)
if commit_action is CommitAction.DEFER_ACTIVE_RESPONSE:
if session.overlap_speech_ms <= session.config.overlap_short_ack_ms:
native.audio_buffer.clear()
session.release_all_input_bytes()
native.input_since_commit = False
native.speech_since_commit = False
native.clear_committed_audio()
if realtime_protocol is not None:
await realtime_protocol.discard_pending_input_audio(
audio_end_ms=session.overlap_speech_ms
)
await emit_event(
{
"type": "input.committed",
"session_id": session.session_id,
"item_id": item_id,
"turn_id": session.turn_id,
"epoch": session.epoch,
"empty": True,
"is_speech": False,
"overlap_ack": True,
"no_response": True,
}
)
session.reset_overlap_speech()
session.discard_response_options()
continue
commit_reservation = native.audio_buffer.prepare_commit(
operation_id=uuid.uuid4().hex,
chunk_period_ms=session.capabilities.chunk_period_ms or 1000,
)
deferred_payload = commit_reservation.payload
if deferred_payload is None:
commit_reservation.commit()
else:
if native.committed_audio_payload is not None:
deferred_payload = self._merge_native_audio_payloads(
native.committed_audio_payload,
deferred_payload,
)
native.retain_committed_audio(
deferred_payload,
operation_id=commit_reservation.operation_id,
reserved_bytes=commit_reservation.byte_count,
)
commit_reservation.commit()
native.deferred_response_create = should_create_response
native.deferred_precreate_response = precreate_response_requested
native.input_since_commit = False
native.speech_since_commit = False
committed = self._commit_native_audio_input(
session,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
)
committed_payload = self._native_audio_committed_payload(
session,
committed=committed,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
)
committed_payload["overlap_deferred"] = True
committed_payload["response_create_deferred"] = should_create_response
await emit_event(committed_payload)
continue
if commit_action is CommitAction.START_AUTO_RESPONSE:
commit_reservation = native.audio_buffer.prepare_commit(
operation_id=uuid.uuid4().hex,
chunk_period_ms=session.capabilities.chunk_period_ms or 1000,
)
committed_input = commit_reservation.payload
final_payload = committed_input
if native.committed_audio_payload is not None:
if final_payload is not None:
final_payload = self._merge_native_audio_payloads(
native.committed_audio_payload,
final_payload,
)
else:
final_payload = native.committed_audio_payload
commit_reservation.commit()
if final_payload is not None:
native.retain_committed_audio(
final_payload,
operation_id=commit_reservation.operation_id,
reserved_bytes=commit_reservation.byte_count,
)
native.deferred_response_create = False
native.input_since_commit = False
native.speech_since_commit = False
data_plane_turn_id = session.turn_id
committed = self._commit_native_audio_input(
session,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
turn_id=data_plane_turn_id,
)
await emit_event(
self._native_audio_committed_payload(
session,
committed=committed,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
)
)
if final_payload is not None:
await start_native_append(
{
**final_payload,
"duplex_turn_id": data_plane_turn_id,
},
final=True,
precreate_response=False,
operation_id=commit_reservation.operation_id,
retained_committed_payload=final_payload,
)
continue
if self._uses_native_input_append(session) and event_type == "response.create":
if (
native_response_in_progress()
or actor.native_append_tasks
or native.data_plane_task is not None
):
if session.active_response_id is None and (
session.active_request_id is not None
or actor.native_append_tasks
or native.data_plane_task is not None
):
continue
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"epoch": session.epoch,
"code": "response_already_active",
"error": "response.create cannot start while another response is active.",
}
)
session.discard_response_options()
continue
if native.committed_audio_payload is not None:
committed_payload = native.committed_audio_payload
operation_id = native.committed_audio_operation_id
if operation_id is None:
operation_id = uuid.uuid4().hex
native.committed_audio_operation_id = operation_id
await start_native_append(
committed_payload,
final=True,
precreate_response=True,
operation_id=operation_id,
retained_committed_payload=committed_payload,
)
continue
await emit_event(
{
"type": "error",
"session_id": session.session_id,
"epoch": session.epoch,
"code": "response_create_without_input",
"error": "Native duplex response.create requires committed audio input.",
}
)
session.discard_response_options()
continue
if self._uses_native_input_append(session) and not native_response_in_progress():
commit_reservation = (
native.audio_buffer.prepare_commit(
operation_id=uuid.uuid4().hex,
chunk_period_ms=session.capabilities.chunk_period_ms or 1000,
)
if event_type in {"input.commit", "input_audio_buffer.commit"}
else None
)
flushed_buffer_reserved_bytes = (
native.audio_buffer.pending_byte_count if commit_reservation is None else 0
)
flushed = (
commit_reservation.payload
if commit_reservation is not None
else native.audio_buffer.flush(chunk_period_ms=session.capabilities.chunk_period_ms or 1000)
)
if native.committed_audio_payload is not None:
if flushed is not None:
flushed = self._merge_native_audio_payloads(
native.committed_audio_payload,
flushed,
)
else:
flushed = native.committed_audio_payload
if commit_reservation is not None:
commit_reservation.commit()
if flushed is not None:
if self._should_force_listen_for_short_commit(session, event, flushed):
flushed = dict(flushed)
flushed["force_listen"] = True
native.input_since_commit = False
committed = self._commit_native_audio_input(
session,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
)
await emit_event(
self._native_audio_committed_payload(
session,
committed=committed,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
)
)
if should_create_response:
native.retain_committed_audio(
flushed,
operation_id=(
commit_reservation.operation_id
if commit_reservation is not None
else uuid.uuid4().hex
),
reserved_bytes=(
commit_reservation.byte_count
if commit_reservation is not None
else flushed_buffer_reserved_bytes
),
)
await start_native_append(
flushed,
final=True,
precreate_response=True,
operation_id=native.committed_audio_operation_id,
retained_committed_payload=flushed,
)
else:
native.retain_committed_audio(
flushed,
operation_id=(
commit_reservation.operation_id
if commit_reservation is not None
else uuid.uuid4().hex
),
reserved_bytes=(
commit_reservation.byte_count
if commit_reservation is not None
else flushed_buffer_reserved_bytes
),
)
native.deferred_response_create = False
continue
native_had_uncommitted_audio = self._uses_native_input_append(session) and (
native.input_since_commit
or native.audio_buffer.has_pending()
or native.committed_audio_payload is not None
or realtime_validated_audio_commit
)
committed = session.commit_user_input()
if self._uses_native_input_append(session) and event_type in {
"input_audio_buffer.commit",
"input.commit",
}:
native.input_since_commit = False
native.speech_since_commit = False
if committed is None and event_type != "response.create":
if self._uses_native_input_append(session):
committed = (
self._commit_native_audio_input(
session,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
)
if native_had_uncommitted_audio
else None
)
await emit_event(
self._native_audio_committed_payload(
session,
committed=committed,
item_id=event.get("item_id"),
transcript=event.get("transcript"),
)
)
else:
await emit_event(
{
"type": "input.committed",
"session_id": session.session_id,
"item_id": item_id,
"empty": True,
}
)
continue
if committed is not None:
item_id = event.get("item_id")
if isinstance(item_id, str):
session.register_history_item(item_id, committed.message)
await emit_event(
self._input_committed_payload(
session,
committed,
item_id=item_id,
)
)
if not should_create_response:
continue
if actor.active_response_task is not None and not actor.active_response_task.done():
await self._cancel_active_response(
session,
actor.active_response_task,
emit_event,
reason="new_response",
)
if session.config.server_vad is not None and not self._uses_native_input_append(session):
start_chat_response()
else:
actor.active_response_task = asyncio.create_task(self._run_response(session, emit_event))
continue
await emit_event(
{
"type": "error",
"error": f"Unknown duplex event: {event_type}",
"code": "unknown_event",
}
)
except WebSocketDisconnect:
logger.info("Duplex session disconnected")
transport_detached = True
except Exception as exc:
logger.exception("Duplex session failed: %s", exc)
with suppress(Exception):
await emit_event({"type": "error", "error": str(exc), "code": "internal_error"})
finally:
if realtime_protocol is not None:
realtime_protocol.reject_realtime_turn_detection_update()
if session is not None:
if reader_task is not None and not reader_task.done():
reader_task.cancel()
with suppress(asyncio.CancelledError):
await reader_task
while pending_turn_reservations > 0:
session.release_pending_turn()
pending_turn_reservations -= 1
resumable_detach = (
transport_detached
and runtime_opened
and not runtime_closed
and session.state == DuplexSessionState.OPEN
and session.capabilities.supports_session_resume
and attachment_generation is not None
and resume_credential_delivered
)
if resumable_detach:
async def cancel_orphan_response_after_grace() -> None:
tasks = self._session_tasks.get(session.session_id)
current_session = self._registry.get(session.session_id)
if tasks is None or current_session is not session or session.state != DuplexSessionState.OPEN:
return
await tasks.cancel_append_tasks(response_bound_only=True)
await self._cancel_native_data_plane_stream(session)
await self._cancel_active_response(
session,
tasks.active_response_task,
emit_event,
reason="disconnect_grace_expired",
notify=False,
)
tasks.active_response_task = None
detached = await self._attachment_registry.detach(
session.session_id,
attachment_generation=attachment_generation,
on_grace_expired=cancel_orphan_response_after_grace,
)
touch_session = getattr(self._chat_service.engine_client, "touch_duplex_session_async", None)
if detached and callable(touch_session):
with suppress(Exception):
await touch_session(
session.session_id,
fence=DuplexFence(
session.session_id,
epoch=session.epoch,
turn_id=session.turn_id,
incarnation=session.incarnation,
),
activity=DuplexLeaseActivity.DETACH,
)
else:
begin_close(actor.close_reason or "disconnect")
await actor.cancel_append_tasks()
await self._cancel_native_data_plane_stream(session)
await self._cancel_active_response(
session,
actor.active_response_task,
emit_event,
reason="disconnect",
notify=False,
)
if runtime_opened and not runtime_closed and session.state != DuplexSessionState.CLOSED:
await self._close_runtime_session(session, reason="disconnect")
self._cleanup_duplex_session_state(session)
self._registry.close(session.session_id)
self._session_tasks.pop(session.session_id, None)
self._realtime_protocols.pop(session.session_id, None)
self._lease_generations.pop(session.session_id, None)
self._resync_required_sessions.discard(session.session_id)
with suppress(Exception):
await self._attachment_registry.close(session.session_id)
self._stop_lifecycle_listener_if_idle()
await actor.close_writer()
with suppress(Exception):
await asyncio.wait_for(actor.output_queue.join(), timeout=2.0)
if not writer_task.done():
writer_task.cancel()
with suppress(asyncio.CancelledError):
await writer_task
Reference in new issue View Git Blame Copy Permalink