[Core][Frontend] Bound duplex output delivery and reject cancelled audio (#7644)

Signed-off-by: Nolen Liang <nliang@nvidia.com>
Signed-off-by: bcsdhjew <nliang@nvidia.com>
This commit is contained in:
bcsdhjew authored and GitHub committed 2026-10-04 14:06:04 -07:00
1 parent 3f20252151
commit f781f7bb08
26 files changed
+1170 -147

No files matched your search

+105 -27
View File
@@ -11,8 +11,9 @@ state.
Control operations (open / close / resume / touch) arrive as correlated RPC
messages and answer through ``result_sink``; session commands arrive one-way
as ``DuplexSessionCommandMessage`` and are pushed onto the runner's ordered
mailbox after backpressure admission; everything a session emits leaves
through ``output_sink`` as ``DuplexSessionEventMessage``.
mailbox after backpressure admission. Public output uses a bounded buffer
shared with that session's consumer. The engine-wide ``output_sink`` only
carries closure notifications and errors without a live session.
"""
from __future__ import annotations
@@ -34,7 +35,16 @@ from vllm_omni.engine.duplex.contracts import (
duplex_resource_request_belongs_to_session,
duplex_resource_request_id,
)
from vllm_omni.engine.duplex.events import DuplexEvent, ErrorEvent, SessionClosed, SessionExpired, error_event
from vllm_omni.engine.duplex.delivery import DuplexOutputBuffer, DuplexOutputOverflowError
from vllm_omni.engine.duplex.events import (
DuplexEvent,
ErrorEvent,
OutputAudioCleared,
ResponseDone,
SessionClosed,
SessionExpired,
error_event,
)
from vllm_omni.engine.duplex.messages import (
CloseDuplexSessionMessage,
DuplexControlResultMessage,
@@ -128,6 +138,8 @@ class DuplexSessionManager:
model_path=getattr(runtime_config, "server_vad_model_path", None)
)
self.runners: dict[str, DuplexSessionRunner] = {}
self._outputs: dict[str, DuplexOutputBuffer] = {}
self._output_failed: set[str] = set()
#: Sessions whose close/expiry began but whose stage cleanup has not finalized;
#: they keep their admission slot until finalized.
self._closing: dict[str, _PendingSessionCleanup] = {}
@@ -270,20 +282,24 @@ class DuplexSessionManager:
if runner is None:
self._emit_raw(
message.session_id,
self._error_event(
"unknown_session",
f"Unknown or closed duplex session: {message.session_id}",
command=command,
),
[
self._error_event(
"unknown_session",
f"Unknown or closed duplex session: {message.session_id}",
command=command,
)
],
)
return
session = runner.session
if runner.closing:
self.emit(
session,
self._error_event(
"session_closed", f"Duplex session is closing: {session.session_id}", command=command
),
[
self._error_event(
"session_closed", f"Duplex session is closing: {session.session_id}", command=command
)
],
)
return
if isinstance(command, AppendAudio):
@@ -293,7 +309,7 @@ class DuplexSessionManager:
if modality_error is not None:
self.emit(
session,
self._error_event("invalid_input_modality", modality_error, command=command),
[self._error_event("invalid_input_modality", modality_error, command=command)],
)
return
limit = int(self.runtime_config.max_pending_input_bytes_per_session)
@@ -301,22 +317,26 @@ class DuplexSessionManager:
if not session.reserve_input_bytes(pending_bytes, limit=limit):
self.emit(
session,
self._error_event(
"input_backpressure",
"Duplex session has too many pending input bytes",
command=command,
),
[
self._error_event(
"input_backpressure",
"Duplex session has too many pending input bytes",
command=command,
)
],
)
return
elif isinstance(command, Commit):
if not session.reserve_pending_turn(limit=int(self.runtime_config.max_pending_turns_per_session)):
self.emit(
session,
self._error_event(
"input_backpressure",
"Duplex session has too many pending input turns",
command=command,
),
[
self._error_event(
"input_backpressure",
"Duplex session has too many pending input turns",
command=command,
)
],
)
return
runner.submit(command)
@@ -329,18 +349,72 @@ class DuplexSessionManager:
# Emission #
# ------------------------------------------------------------------ #
def emit(self, session: DuplexEngineSession, event: DuplexEvent) -> None:
"""Bind the session identity to one typed event and push it to the engine output queue."""
self._emit_raw(session.session_id, event, epoch=session.epoch)
def emit(self, session: DuplexEngineSession, events: list[DuplexEvent]) -> None:
"""Bind identity and deliver through the session's bounded output."""
self._emit_raw(session.session_id, events, epoch=session.epoch)
def invalidate_output(self, session_id: str, response_id: str | None, *, through_epoch: int) -> None:
output = self._outputs.get(session_id)
if output is not None and response_id is not None:
output.invalidate(response_id, through_epoch=through_epoch)
def _emit_raw(
self,
session_id: str,
event: DuplexEvent,
events: list[DuplexEvent],
*,
epoch: int | None = None,
) -> None:
ending_delivered = False
for index, event in enumerate(events):
try:
self._deliver(session_id, event, epoch=epoch)
except DuplexOutputOverflowError as exc:
if session_id in self._output_failed:
return
self._output_failed.add(session_id)
runner = self.runners.get(session_id)
if runner is None:
return
# Projection has already marked this batch's ending as emitted.
# Retain it instead of recursively projecting a second ending.
terminal = next((item for item in events[index:] if isinstance(item, ResponseDone)), None)
response_id = terminal.response_id if terminal is not None else event.response_id
if response_id is None and not ending_delivered:
response_id = runner.session.active_response_id
for recovery in runner.fail_output(str(exc), response_id=response_id, terminal=terminal):
try:
self._deliver(session_id, recovery, epoch=epoch)
except DuplexOutputOverflowError:
# A full control reserve may omit an error/response ending;
# session closure still has its independent terminal slot.
pass
return
ending_delivered |= isinstance(event, ResponseDone)
def _deliver(self, session_id: str, event: DuplexEvent, *, epoch: int | None) -> None:
event = replace(event, session_id=session_id, epoch=epoch)
output = self._outputs.get(session_id)
if output is not None and not isinstance(event, SessionClosed):
if isinstance(event, OutputAudioCleared) and epoch is not None:
self.invalidate_output(session_id, event.response_id, through_epoch=epoch)
if session_id in self._output_failed:
# The failure/close path has a finite reserve; stop adding ordinary output.
if not isinstance(event, ErrorEvent) and not (
isinstance(event, ResponseDone) and event.status in {"failed", "cancelled"}
):
return
output.put(event)
return
if isinstance(event, SessionClosed):
self._outputs.pop(session_id, None)
self._output_failed.discard(session_id)
elif not isinstance(event, ErrorEvent):
# A closed/rolled-back session must not move late audio back into
# the unbounded shared queue after its output buffer is removed.
return
# Only closure notifications and errors without a live session use the
# engine-wide queue. Audio never accumulates there before the bounded buffer.
message = DuplexSessionEventMessage(session_id=session_id, event=event)
put_nowait = getattr(self._output_sink, "put_nowait", None)
if callable(put_nowait):
@@ -514,6 +588,7 @@ class DuplexSessionManager:
model_config=self.model_config,
)
self.runners[session_id] = runner
self._outputs[session_id] = message.output_buffer
# Reserve the Stage0 request resource atomically with admission.
self.ensure_stage_request(session, stage_id=0)
runner.start()
@@ -566,6 +641,7 @@ class DuplexSessionManager:
owned_runner = runner is not None and self.runners.get(session_id) is runner
if owned_runner:
self.runners.pop(session_id, None)
self._outputs.pop(session_id, None)
pending: _PendingRequestCleanup | None = None
key: tuple[str, int] | None = None
if session is not None:
@@ -686,7 +762,7 @@ class DuplexSessionManager:
terminal: DuplexEvent = (
SessionExpired(reason=reason) if kind == "expired" else SessionClosed(reason=reason)
)
self.emit(session, terminal)
self.emit(session, [terminal])
async def _finalize_pending_cleanup(
self,
@@ -943,6 +1019,8 @@ class DuplexSessionManager:
self._closing.clear()
self._session_snapshots.clear()
self._request_index.clear()
self._outputs.clear()
self._output_failed.clear()
if self._owns_executor:
self.executor.shutdown(wait=False, cancel_futures=True)