[Core][Frontend] Bound duplex output delivery and reject cancelled audio (#7644)
Signed-off-by: Nolen Liang <nliang@nvidia.com> Signed-off-by: bcsdhjew <nliang@nvidia.com>
This commit is contained in:
1 parent
3f20252151
commit
f781f7bb08
26 files changed
+1170
-147
No files matched your search
@@ -11,8 +11,9 @@ state.
|
||||
Control operations (open / close / resume / touch) arrive as correlated RPC
|
||||
messages and answer through ``result_sink``; session commands arrive one-way
|
||||
as ``DuplexSessionCommandMessage`` and are pushed onto the runner's ordered
|
||||
mailbox after backpressure admission; everything a session emits leaves
|
||||
through ``output_sink`` as ``DuplexSessionEventMessage``.
|
||||
mailbox after backpressure admission. Public output uses a bounded buffer
|
||||
shared with that session's consumer. The engine-wide ``output_sink`` only
|
||||
carries closure notifications and errors without a live session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -34,7 +35,16 @@ from vllm_omni.engine.duplex.contracts import (
|
||||
duplex_resource_request_belongs_to_session,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.engine.duplex.events import DuplexEvent, ErrorEvent, SessionClosed, SessionExpired, error_event
|
||||
from vllm_omni.engine.duplex.delivery import DuplexOutputBuffer, DuplexOutputOverflowError
|
||||
from vllm_omni.engine.duplex.events import (
|
||||
DuplexEvent,
|
||||
ErrorEvent,
|
||||
OutputAudioCleared,
|
||||
ResponseDone,
|
||||
SessionClosed,
|
||||
SessionExpired,
|
||||
error_event,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
CloseDuplexSessionMessage,
|
||||
DuplexControlResultMessage,
|
||||
@@ -128,6 +138,8 @@ class DuplexSessionManager:
|
||||
model_path=getattr(runtime_config, "server_vad_model_path", None)
|
||||
)
|
||||
self.runners: dict[str, DuplexSessionRunner] = {}
|
||||
self._outputs: dict[str, DuplexOutputBuffer] = {}
|
||||
self._output_failed: set[str] = set()
|
||||
#: Sessions whose close/expiry began but whose stage cleanup has not finalized;
|
||||
#: they keep their admission slot until finalized.
|
||||
self._closing: dict[str, _PendingSessionCleanup] = {}
|
||||
@@ -270,20 +282,24 @@ class DuplexSessionManager:
|
||||
if runner is None:
|
||||
self._emit_raw(
|
||||
message.session_id,
|
||||
self._error_event(
|
||||
"unknown_session",
|
||||
f"Unknown or closed duplex session: {message.session_id}",
|
||||
command=command,
|
||||
),
|
||||
[
|
||||
self._error_event(
|
||||
"unknown_session",
|
||||
f"Unknown or closed duplex session: {message.session_id}",
|
||||
command=command,
|
||||
)
|
||||
],
|
||||
)
|
||||
return
|
||||
session = runner.session
|
||||
if runner.closing:
|
||||
self.emit(
|
||||
session,
|
||||
self._error_event(
|
||||
"session_closed", f"Duplex session is closing: {session.session_id}", command=command
|
||||
),
|
||||
[
|
||||
self._error_event(
|
||||
"session_closed", f"Duplex session is closing: {session.session_id}", command=command
|
||||
)
|
||||
],
|
||||
)
|
||||
return
|
||||
if isinstance(command, AppendAudio):
|
||||
@@ -293,7 +309,7 @@ class DuplexSessionManager:
|
||||
if modality_error is not None:
|
||||
self.emit(
|
||||
session,
|
||||
self._error_event("invalid_input_modality", modality_error, command=command),
|
||||
[self._error_event("invalid_input_modality", modality_error, command=command)],
|
||||
)
|
||||
return
|
||||
limit = int(self.runtime_config.max_pending_input_bytes_per_session)
|
||||
@@ -301,22 +317,26 @@ class DuplexSessionManager:
|
||||
if not session.reserve_input_bytes(pending_bytes, limit=limit):
|
||||
self.emit(
|
||||
session,
|
||||
self._error_event(
|
||||
"input_backpressure",
|
||||
"Duplex session has too many pending input bytes",
|
||||
command=command,
|
||||
),
|
||||
[
|
||||
self._error_event(
|
||||
"input_backpressure",
|
||||
"Duplex session has too many pending input bytes",
|
||||
command=command,
|
||||
)
|
||||
],
|
||||
)
|
||||
return
|
||||
elif isinstance(command, Commit):
|
||||
if not session.reserve_pending_turn(limit=int(self.runtime_config.max_pending_turns_per_session)):
|
||||
self.emit(
|
||||
session,
|
||||
self._error_event(
|
||||
"input_backpressure",
|
||||
"Duplex session has too many pending input turns",
|
||||
command=command,
|
||||
),
|
||||
[
|
||||
self._error_event(
|
||||
"input_backpressure",
|
||||
"Duplex session has too many pending input turns",
|
||||
command=command,
|
||||
)
|
||||
],
|
||||
)
|
||||
return
|
||||
runner.submit(command)
|
||||
@@ -329,18 +349,72 @@ class DuplexSessionManager:
|
||||
# Emission #
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def emit(self, session: DuplexEngineSession, event: DuplexEvent) -> None:
|
||||
"""Bind the session identity to one typed event and push it to the engine output queue."""
|
||||
self._emit_raw(session.session_id, event, epoch=session.epoch)
|
||||
def emit(self, session: DuplexEngineSession, events: list[DuplexEvent]) -> None:
|
||||
"""Bind identity and deliver through the session's bounded output."""
|
||||
self._emit_raw(session.session_id, events, epoch=session.epoch)
|
||||
|
||||
def invalidate_output(self, session_id: str, response_id: str | None, *, through_epoch: int) -> None:
|
||||
output = self._outputs.get(session_id)
|
||||
if output is not None and response_id is not None:
|
||||
output.invalidate(response_id, through_epoch=through_epoch)
|
||||
|
||||
def _emit_raw(
|
||||
self,
|
||||
session_id: str,
|
||||
event: DuplexEvent,
|
||||
events: list[DuplexEvent],
|
||||
*,
|
||||
epoch: int | None = None,
|
||||
) -> None:
|
||||
ending_delivered = False
|
||||
for index, event in enumerate(events):
|
||||
try:
|
||||
self._deliver(session_id, event, epoch=epoch)
|
||||
except DuplexOutputOverflowError as exc:
|
||||
if session_id in self._output_failed:
|
||||
return
|
||||
self._output_failed.add(session_id)
|
||||
runner = self.runners.get(session_id)
|
||||
if runner is None:
|
||||
return
|
||||
# Projection has already marked this batch's ending as emitted.
|
||||
# Retain it instead of recursively projecting a second ending.
|
||||
terminal = next((item for item in events[index:] if isinstance(item, ResponseDone)), None)
|
||||
response_id = terminal.response_id if terminal is not None else event.response_id
|
||||
if response_id is None and not ending_delivered:
|
||||
response_id = runner.session.active_response_id
|
||||
for recovery in runner.fail_output(str(exc), response_id=response_id, terminal=terminal):
|
||||
try:
|
||||
self._deliver(session_id, recovery, epoch=epoch)
|
||||
except DuplexOutputOverflowError:
|
||||
# A full control reserve may omit an error/response ending;
|
||||
# session closure still has its independent terminal slot.
|
||||
pass
|
||||
return
|
||||
ending_delivered |= isinstance(event, ResponseDone)
|
||||
|
||||
def _deliver(self, session_id: str, event: DuplexEvent, *, epoch: int | None) -> None:
|
||||
event = replace(event, session_id=session_id, epoch=epoch)
|
||||
output = self._outputs.get(session_id)
|
||||
if output is not None and not isinstance(event, SessionClosed):
|
||||
if isinstance(event, OutputAudioCleared) and epoch is not None:
|
||||
self.invalidate_output(session_id, event.response_id, through_epoch=epoch)
|
||||
if session_id in self._output_failed:
|
||||
# The failure/close path has a finite reserve; stop adding ordinary output.
|
||||
if not isinstance(event, ErrorEvent) and not (
|
||||
isinstance(event, ResponseDone) and event.status in {"failed", "cancelled"}
|
||||
):
|
||||
return
|
||||
output.put(event)
|
||||
return
|
||||
if isinstance(event, SessionClosed):
|
||||
self._outputs.pop(session_id, None)
|
||||
self._output_failed.discard(session_id)
|
||||
elif not isinstance(event, ErrorEvent):
|
||||
# A closed/rolled-back session must not move late audio back into
|
||||
# the unbounded shared queue after its output buffer is removed.
|
||||
return
|
||||
# Only closure notifications and errors without a live session use the
|
||||
# engine-wide queue. Audio never accumulates there before the bounded buffer.
|
||||
message = DuplexSessionEventMessage(session_id=session_id, event=event)
|
||||
put_nowait = getattr(self._output_sink, "put_nowait", None)
|
||||
if callable(put_nowait):
|
||||
@@ -514,6 +588,7 @@ class DuplexSessionManager:
|
||||
model_config=self.model_config,
|
||||
)
|
||||
self.runners[session_id] = runner
|
||||
self._outputs[session_id] = message.output_buffer
|
||||
# Reserve the Stage0 request resource atomically with admission.
|
||||
self.ensure_stage_request(session, stage_id=0)
|
||||
runner.start()
|
||||
@@ -566,6 +641,7 @@ class DuplexSessionManager:
|
||||
owned_runner = runner is not None and self.runners.get(session_id) is runner
|
||||
if owned_runner:
|
||||
self.runners.pop(session_id, None)
|
||||
self._outputs.pop(session_id, None)
|
||||
pending: _PendingRequestCleanup | None = None
|
||||
key: tuple[str, int] | None = None
|
||||
if session is not None:
|
||||
@@ -686,7 +762,7 @@ class DuplexSessionManager:
|
||||
terminal: DuplexEvent = (
|
||||
SessionExpired(reason=reason) if kind == "expired" else SessionClosed(reason=reason)
|
||||
)
|
||||
self.emit(session, terminal)
|
||||
self.emit(session, [terminal])
|
||||
|
||||
async def _finalize_pending_cleanup(
|
||||
self,
|
||||
@@ -943,6 +1019,8 @@ class DuplexSessionManager:
|
||||
self._closing.clear()
|
||||
self._session_snapshots.clear()
|
||||
self._request_index.clear()
|
||||
self._outputs.clear()
|
||||
self._output_failed.clear()
|
||||
if self._owns_executor:
|
||||
self.executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
|
||||
Reference in new issue
Block a user