[Bugfix][Frontend] Preserve duplex chat audio format and duration (#7316)
Signed-off-by: Nolen Liang <nliang@nvidia.com>
This commit is contained in:
1 parent
1b6cd282f0
commit
e0c4f97f32
13 files changed
+684
-40
No files matched your search
@@ -7,6 +7,7 @@ import asyncio
|
||||
import binascii
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from copy import deepcopy
|
||||
|
||||
@@ -99,7 +100,9 @@ class DuplexSessionRunnerMixin:
|
||||
if callable(close):
|
||||
await close(code=1000, reason=reason)
|
||||
|
||||
async def send_attachment_payload(payload: dict[str, object], *, journal: bool) -> None:
|
||||
async def send_attachment_payload(
|
||||
payload: dict[str, object], *, journal: bool, on_accepted: Callable[[], None] | None = None
|
||||
) -> None:
|
||||
assert session is not None
|
||||
session_id = session.session_id
|
||||
should_journal = journal and session_id not in self._resync_required_sessions
|
||||
@@ -108,6 +111,7 @@ class DuplexSessionRunnerMixin:
|
||||
session_id,
|
||||
payload,
|
||||
journal=should_journal,
|
||||
on_accepted=on_accepted,
|
||||
)
|
||||
except DuplexJournalOverflowError:
|
||||
first_overflow = session_id not in self._resync_required_sessions
|
||||
@@ -129,6 +133,7 @@ class DuplexSessionRunnerMixin:
|
||||
session_id,
|
||||
payload,
|
||||
journal=False,
|
||||
on_accepted=on_accepted,
|
||||
)
|
||||
|
||||
if realtime_protocol is not None:
|
||||
@@ -155,23 +160,29 @@ class DuplexSessionRunnerMixin:
|
||||
|
||||
event_emit_lock = asyncio.Lock()
|
||||
|
||||
async def send_outbound(payload: dict[str, object]) -> None:
|
||||
async def send_outbound(payload: dict[str, object], *, on_accepted: Callable[[], None] | None = None) -> None:
|
||||
if not attachment_ready or session is None:
|
||||
await actor.send_json(dict(payload))
|
||||
if on_accepted is not None:
|
||||
on_accepted()
|
||||
return
|
||||
if realtime_protocol is None:
|
||||
await send_attachment_payload(
|
||||
payload,
|
||||
journal=payload.get("type") not in {"session.created", "session.resumed"},
|
||||
on_accepted=on_accepted,
|
||||
)
|
||||
return
|
||||
for projected in realtime_protocol.encode_outbound_event(payload):
|
||||
await send_attachment_payload(
|
||||
projected,
|
||||
journal=projected.get("type") not in {"session.created", "session.resumed"},
|
||||
# Item/part announcements may precede the audio. Only the
|
||||
# actual audio event establishes acceptance of this chunk.
|
||||
on_accepted=on_accepted if projected.get("type") == "response.audio.delta" else None,
|
||||
)
|
||||
|
||||
async def emit_event(payload: dict[str, object]) -> None:
|
||||
async def emit_event(payload: dict[str, object], *, on_accepted: Callable[[], None] | None = None) -> None:
|
||||
deferred_precreate_response = False
|
||||
async with event_emit_lock:
|
||||
accepted, deferred_overlap_payload = await self._apply_outbound_session_event(
|
||||
@@ -183,7 +194,7 @@ class DuplexSessionRunnerMixin:
|
||||
)
|
||||
if not accepted:
|
||||
return
|
||||
await send_outbound(payload)
|
||||
await send_outbound(payload, on_accepted=on_accepted)
|
||||
if deferred_overlap_payload is not None:
|
||||
deferred_precreate_response = native.deferred_precreate_response
|
||||
native.deferred_precreate_response = False
|
||||
|
||||
Reference in new issue
Block a user