[Model] Integrate AURA into Full-Duplex Runtime (#7633)

This commit is contained in:
NumberWan authored and GitHub committed 2026-09-21 09:26:54 -07:00
1 parent d397a2f547
commit 62b15142bd
77 files changed
+7695 -457

No files matched your search

@@ -17,7 +17,7 @@ from __future__ import annotations
import copy
import time
from collections.abc import Callable, Mapping
from collections.abc import Callable, Iterable, Mapping
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING, Any
@@ -105,6 +105,9 @@ class ResponseState:
active_response_input_commit_seq: int | None = None
active_response_awaits_input_commit: bool = False
last_response_id: str | None = None
#: Overlapped-input: prior-turn Stage2/3 request ids still draining under
#: their own ``response_id`` after the next turn opened a new response.
draining_response_by_request: dict[str, str] = field(default_factory=dict)
assistant_text_buffer: list[str] = field(default_factory=list)
assistant_audio_text_marks: list[DuplexAssistantAudioTextMark] = field(default_factory=list)
pending_options: ResponseCreateOptions | None = None
@@ -372,6 +375,28 @@ class DuplexEngineSession:
}
return stale
def release_resources_for_request_ids(self, request_ids: Iterable[str]) -> list[str]:
"""Drop bindings for these request ids, whichever fence they sit on.
``cancel_fence`` only releases the fence being cancelled. Overlapped
draining output stages belong to an older turn fence and would
otherwise stay until the session closes.
"""
wanted = {request_id for request_id in request_ids if isinstance(request_id, str) and request_id}
if not wanted:
return []
released = list(
dict.fromkeys(
resource.request_id for resource in self.request_resources.values() if resource.request_id in wanted
)
)
self.request_resources = {
resource_key: resource
for resource_key, resource in self.request_resources.items()
if resource.request_id not in wanted
}
return released
def cancel_fence(self, cancelled_fence: DuplexFence, next_fence: DuplexFence) -> list[str]:
stale = self.prepare_cancel_fence(cancelled_fence, next_fence)
self.release_fence(cancelled_fence)
@@ -587,8 +612,51 @@ class DuplexEngineSession:
return False
if turn_id is None:
return True
# Overlapped input opens a new response_id per released turn; each
# response only accepts its own turn. Draining prior TTS is keyed by
# request_id → response_id, not by this guard.
active_turn_id = self._response.active_response_turn_id
return active_turn_id is None or int(turn_id) == active_turn_id
if active_turn_id is None or int(turn_id) == int(active_turn_id):
return True
return False
def bind_draining_request(self, request_id: str, response_id: str) -> None:
"""Map a still-playing draining-stage request onto the response that owns it."""
if request_id and response_id:
self._response.draining_response_by_request[request_id] = response_id
def response_id_for_request(self, request_id: str | None) -> str | None:
if isinstance(request_id, str) and request_id in self._response.draining_response_by_request:
return self._response.draining_response_by_request[request_id]
return self._response.active_response_id
def pop_draining_request(self, request_id: str | None) -> str | None:
if not isinstance(request_id, str):
return None
return self._response.draining_response_by_request.pop(request_id, None)
def is_draining_request(self, request_id: str | None) -> bool:
return isinstance(request_id, str) and request_id in self._response.draining_response_by_request
def response_has_draining_request(self, response_id: str | None) -> bool:
if response_id is None:
return False
return any(resp == response_id for resp in self._response.draining_response_by_request.values())
def draining_request_ids(self) -> list[str]:
"""Request ids whose TTS is still running under a previous response."""
return list(self._response.draining_response_by_request)
def clear_draining_requests(self) -> None:
self._response.draining_response_by_request.clear()
def clear_draining_for_response(self, response_id: str | None) -> None:
"""Drop draining bindings owned by ``response_id``; leave other responses intact."""
if response_id is None:
return
self._response.draining_response_by_request = {
rid: resp for rid, resp in self._response.draining_response_by_request.items() if resp != response_id
}
def append_history_message(self, message: dict[str, object]) -> None:
self._conversation.messages.append(message)
@@ -707,6 +775,57 @@ class DuplexEngineSession:
self._response.active_options = None
self._response.pending_options = None
def snapshot_active_response_for_drain(self) -> None:
"""Keep the active response ACK-admissible after a later response starts.
Overlapped commit calls ``begin_response`` while prior TTS is still
draining. That clears the live text buffer and playback cursor.
``end_response`` is the normal snapshot, but it would also close the
response. Copy the same snapshot (and reserve the history slot) first
so a playback ACK for the draining response is not
``playback_item_not_found``.
"""
response_id = self.active_response_id
if response_id is None:
return
if response_id not in self._playback.by_response:
self._playback.by_response[response_id] = self._playback.current
assistant_text = "".join(self._response.assistant_text_buffer).strip()
message = _object_dict(role="assistant", content=assistant_text)
seq = self._response.active_response_input_commit_seq
if seq is None:
seq = self.input_commit_seq
self._conversation.assistant_response_snapshots[response_id] = (
copy.deepcopy(message),
tuple(copy.deepcopy(self._response.assistant_audio_text_marks)),
seq,
)
self.reserve_history_item(f"item_{response_id}")
def append_draining_assistant_text(self, response_id: str, text: str) -> None:
"""Append text onto a snapshotted draining response, not the active one."""
if not text:
return
snapshot = self._conversation.assistant_response_snapshots.get(response_id)
if snapshot is None:
return
message, marks, seq = snapshot
content = message.get("content")
if not isinstance(content, str):
content = ""
message["content"] = content + text
self._conversation.assistant_response_snapshots[response_id] = (message, marks, seq)
def assistant_transcript(self, response_id: str | None = None) -> str:
"""Joined assistant text for the active response, or a draining snapshot."""
if response_id is not None and response_id != self.active_response_id:
snapshot = self._conversation.assistant_response_snapshots.get(response_id)
if snapshot is None:
return ""
content = snapshot[0].get("content")
return content if isinstance(content, str) else ""
return "".join(self._response.assistant_text_buffer)
def begin_response(self, *, turn_id: int | None = None) -> str:
self._activate_response_options()
response_id = f"resp-{self.session_id}-{self.epoch}-{uuid4().hex[:8]}"
@@ -881,7 +1000,18 @@ class DuplexEngineSession:
audio_text_marks: list[dict[str, object]] | None = None,
text_requires_complete_audio: bool = False,
audio_complete: bool = False,
response_id: str | None = None,
) -> None:
if response_id is not None and response_id != self.active_response_id:
self._mark_draining_audio_sent(
response_id,
duration_ms,
text_chars=text_chars,
audio_text_marks=audio_text_marks,
text_requires_complete_audio=text_requires_complete_audio,
audio_complete=audio_complete,
)
return
playback = self._playback.current
playback.text_requires_complete_audio |= text_requires_complete_audio
playback.audio_complete |= audio_complete
@@ -911,6 +1041,52 @@ class DuplexEngineSession:
)
self.turn_state = DuplexTurnState.ASSISTANT_PLAYING
def _mark_draining_audio_sent(
self,
response_id: str,
duration_ms: int | None,
*,
text_chars: int | None,
audio_text_marks: list[dict[str, object]] | None,
text_requires_complete_audio: bool,
audio_complete: bool,
) -> None:
"""Attribute audio to a draining response, not the newly active cursor."""
playback = self._playback_cursor_for_response(response_id)
playback.text_requires_complete_audio |= text_requires_complete_audio
playback.audio_complete |= audio_complete
marks: list[DuplexAssistantAudioTextMark] = []
snapshot = self._conversation.assistant_response_snapshots.get(response_id)
if snapshot is not None:
marks = list(snapshot[1])
if duration_ms is not None:
playback.generated_ms = max(playback.generated_ms, duration_ms)
playback.sent_ms = max(playback.sent_ms, duration_ms)
if text_chars is not None and text_chars >= 0:
marks.append(
DuplexAssistantAudioTextMark(
text_chars=int(text_chars),
audio_end_ms=max(0, int(duration_ms)),
)
)
if audio_text_marks:
for raw_mark in audio_text_marks:
if not isinstance(raw_mark, dict):
continue
raw_text_chars = raw_mark.get("text_chars")
raw_audio_end_ms = raw_mark.get("audio_end_ms", raw_mark.get("audio_ms"))
if not isinstance(raw_text_chars, int | float) or not isinstance(raw_audio_end_ms, int | float):
continue
marks.append(
DuplexAssistantAudioTextMark(
text_chars=max(0, int(raw_text_chars)),
audio_end_ms=max(0, int(raw_audio_end_ms)),
)
)
if snapshot is not None:
message, _, seq = snapshot
self._conversation.assistant_response_snapshots[response_id] = (message, tuple(marks), seq)
def _playback_cursor_for_response(self, response_id: str | None = None) -> DuplexPlaybackCursor:
if response_id is None:
return self._playback.current
@@ -951,6 +1127,27 @@ class DuplexEngineSession:
return
self._playback.by_response.pop(response_id, None)
def release_finished_drain_response(self, response_id: str | None) -> None:
"""Drop unused books for a response whose TTS just finished.
Overlap leaves the next turn active, so this must not call
``end_response``. Audio that was already sent stays ACK-admissible:
the client may ``playback.ack`` after the drain ``response.done``.
A response that sent no audio has nothing to acknowledge, so its
snapshot, playback cursor, and unused history placeholder are dropped.
The live response is left untouched.
"""
if response_id is None or response_id == self.active_response_id:
return
playback = self._playback.by_response.get(response_id)
if playback is not None and max(playback.sent_ms, playback.generated_ms) > 0:
return
self._conversation.assistant_response_snapshots.pop(response_id, None)
self.release_response_playback(response_id)
item_id = f"item_{response_id}"
if item_id not in self._conversation.item_ids:
self._discard_history_item_placeholder(item_id)
def clear_playback_cursor(self) -> None:
self._playback.current = DuplexPlaybackCursor()
if self.active_response_id is not None:
@@ -1010,6 +1207,8 @@ class DuplexEngineSession:
self._response.active_response_turn_id = None
self._response.active_response_input_commit_seq = None
self._response.active_response_awaits_input_commit = False
# Keep draining bindings for other responses (older TTS may still play).
self.clear_draining_for_response(response_id)
self._clear_response_metrics()
self.turn_state = DuplexTurnState.IDLE
self._restore_response_config()
@@ -1344,6 +1543,7 @@ class DuplexEngineSession:
self._response.active_response_turn_id = None
self._response.active_response_input_commit_seq = None
self._response.active_response_awaits_input_commit = False
self._response.draining_response_by_request.clear()
self._clear_response_metrics()
self._restore_response_config()
self.turn_state = DuplexTurnState.BARGE_IN