[Bugfix] Compensate cancelled duplex open, append and resume; fix serving README drift (#7636 Issues 1, 2, 3, 21) (#7785)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-09-24 19:23:08 -07:00
1 parent 551a03102f
commit 9e7e3a802c
19 files changed
+1697 -96

No files matched your search

+168 -23
View File
@@ -36,11 +36,13 @@ from vllm_omni.engine.duplex.events import (
SessionResyncRequired,
)
from vllm_omni.engine.duplex.messages import DuplexSessionError
from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, parse_resume_request
from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, ResumeRequest, parse_resume_request
from vllm_omni.entrypoints.duplex.session_attachment import (
DuplexDetachedAttachment,
DuplexJournalGapError,
DuplexJournalOverflowError,
DuplexSessionAttachmentRegistry,
DuplexSessionResumeResult,
InvalidResumeTokenError,
ResumeToken,
)
@@ -67,6 +69,12 @@ _PUMP_DRAIN_TIMEOUT_S = 5.0
_UNJOURNALED_EVENTS = (SessionCreated,)
def _log_rollback_failure(done: asyncio.Task[None]) -> None:
"""A shielded resume rollback outlives its caller; its failure must still be seen."""
if not done.cancelled() and done.exception() is not None:
logger.error("duplex resume rollback %s failed: %r", done.get_name(), done.exception())
@dataclass
class _Attachment:
"""What one websocket connection knows about the session it is attached to."""
@@ -199,10 +207,14 @@ class OmniDuplexSessionHandler:
await send_json(envelope.error_payload("invalid_session_config", str(exc)))
return None
attachment_send, attachment_close = attachment_callbacks(websocket)
# The registry keeps the engine lease generation each attachment is
# serving, so every detach is fenced on the dropped connection's own
# lease (see ``release_attachment``).
created = await self._attachment_registry.create(
handle.session_id,
send=attachment_send,
close=attachment_close,
lease_generation=handle.lease_generation,
)
resume_supported = bool(handle.capabilities.supports_session_resume)
attachment = _Attachment(handle=handle, generation=created.attachment_generation)
@@ -258,14 +270,44 @@ class OmniDuplexSessionHandler:
except (KeyError, ValueError) as exc:
await send_json(envelope.error_payload("session_resume_conflict", str(exc)))
return None
# The claim tells the registry a resume is about to activate: a lease
# generation orphaned meanwhile (another resume abandoned mid-RPC) is
# parked for this activation instead of being handed to the socket it
# is about to replace. The claim ends whatever happens below, and an
# orphan nobody else will serve is then ours to put into grace.
await self._attachment_registry.begin_resume(session_id)
try:
return await self._resume_claimed(websocket, envelope, request, handle, send_json)
finally:
await self._end_resume_claim(session_id)
async def _resume_claimed(
self,
websocket: WebSocket,
envelope: RealtimeEnvelope,
request: ResumeRequest,
handle: DuplexSessionHandle,
send_json: SendJson,
) -> _Attachment | None:
session_id = request.session_id
try:
# If this task is cancelled while the RPC is in flight, DuplexOmni
# observes the outcome and settles a resume that landed through
# ``_settle_abandoned_resume``: the generation goes to the resume
# waiting to activate, else to whoever is attached, else the lease
# goes back into disconnect grace.
await self._omni.resume_session(
session_id,
expected_lease_generation=handle.lease_generation,
on_abandoned=lambda lease_generation: self._settle_abandoned_resume(session_id, lease_generation),
)
except DuplexSessionError as exc:
await send_json(envelope.error_payload("runtime_resume_failed", str(exc)))
return None
# The generation this resume produced, read before anything else can
# run: the handle is shared by every connection of the session, and a
# concurrent resume adopts its own, newer generation into it.
lease_generation = handle.lease_generation
attachment_send, attachment_close = attachment_callbacks(websocket)
@@ -277,6 +319,12 @@ class OmniDuplexSessionHandler:
resume_token=token.plaintext,
).to_realtime()
# From here on the engine lease is resumed (``detached_at`` cleared), so
# every exit that does not hand the attachment to the caller has to
# put the lease back into its disconnect grace, or the session sits
# attached to nothing until idle expiry. That includes cancellation of
# the handler task, which is not an ``Exception``.
resumed: DuplexSessionResumeResult | None = None
try:
resumed = await self._attachment_registry.resume(
session_id,
@@ -285,26 +333,27 @@ class OmniDuplexSessionHandler:
send=attachment_send,
close=attachment_close,
activation_payload_factory=activation_payload_factory,
lease_generation=lease_generation,
)
replaced = resumed.replaced_attachment
if replaced is not None:
with suppress(Exception):
await replaced.send(
SessionReplaced(session_id=session_id, attachment_generation=replaced.generation).to_realtime()
)
with suppress(Exception):
await replaced.close("session_replaced")
except asyncio.CancelledError:
await self._abandon_resume(
session_id,
attachment_generation=resumed.attachment_generation if resumed is not None else None,
lease_generation=lease_generation,
)
raise
except Exception as exc:
# The engine lease was already resumed above, so a failed activation
# would leave the session attached to nothing -- unless another
# connection won the race and is attached right now. Detaching then
# would start the disconnect grace for the *winner*, which ordinary
# heartbeats do not clear. Roll back only what this attempt owns.
if not await self._attachment_registry.has_attachment(session_id):
with suppress(DuplexSessionError):
await self._omni.detach_session(session_id)
await self._abandon_resume(session_id, attachment_generation=None, lease_generation=lease_generation)
await send_json(envelope.error_payload("session_resume_conflict", str(exc)))
return None
replaced = resumed.replaced_attachment
if replaced is not None:
with suppress(Exception):
await replaced.send(
SessionReplaced(session_id=session_id, attachment_generation=replaced.generation).to_realtime()
)
with suppress(Exception):
await replaced.close("session_replaced")
# A reconnect brings a fresh envelope carrying pcm16/16 kHz wire
# defaults. The negotiated input format is a wire default, not part of
# the public session object, so it has to be carried over explicitly:
@@ -315,6 +364,96 @@ class OmniDuplexSessionHandler:
self._start_pump(handle, None)
return _Attachment(handle=handle, generation=resumed.attachment_generation)
async def _abandon_resume(
self,
session_id: str,
*,
attachment_generation: int | None,
lease_generation: int,
) -> None:
"""Roll the engine lease of a resume this connection will never serve back into disconnect grace.
Only what this attempt owns is rolled back. ``attachment_generation``
is the attachment it activated, or ``None`` when activation did not
happen (or was rolled back by the registry): then the generation this
attempt's engine resume produced is settled like an abandoned resume
(``_settle_abandoned_resume``). The registry knows more than this
method does: a rolled-back activation may have been handed a newer
generation meanwhile, which the registry parks as the orphan, and the
claim ending in ``_resume`` gives it back if nobody else will serve
it. Every engine detach is fenced on the generation being given up, so
a newer lease a concurrent resume produced is never touched.
The rollback runs shielded: a second cancellation landing between the
registry detach and the engine detach would otherwise strand the
session exactly the way this method compensates for.
"""
rollback = asyncio.ensure_future(
self._roll_back_resume(
session_id,
attachment_generation=attachment_generation,
lease_generation=lease_generation,
)
)
rollback.add_done_callback(_log_rollback_failure)
await asyncio.shield(rollback)
async def _roll_back_resume(
self,
session_id: str,
*,
attachment_generation: int | None,
lease_generation: int,
) -> None:
if attachment_generation is None:
await self._settle_abandoned_resume(session_id, lease_generation)
return
released = await self._attachment_registry.release_attachment(
session_id, attachment_generation=attachment_generation
)
if released is not None:
await self._detach_lease(released)
async def _settle_abandoned_resume(self, session_id: str, lease_generation: int) -> None:
"""Settle the engine lease generation of a resume its connection never got to serve.
The engine already bumped the lease to ``lease_generation`` and
cleared its disconnect grace. The registry picks the owner: a resume
waiting to activate (it will serve the session), else the connection
attached right now (a takeover that never activated leaves the
previous socket serving, and its own disconnect must be able to
detach this lease), else nobody, and then the lease goes back into
disconnect grace here, fenced on the generation so a later resume is
never detached.
"""
orphan = await self._attachment_registry.settle_lease_generation(session_id, lease_generation)
if orphan is not None:
with suppress(DuplexSessionError):
await self._omni.detach_session(session_id, expected_lease_generation=orphan)
async def _end_resume_claim(self, session_id: str) -> None:
"""End this connection's resume claim; detach an orphaned lease nobody else will serve.
Runs from a ``finally``, so possibly during cancellation: the detach
is shielded like the other rollbacks.
"""
orphan = await self._attachment_registry.end_resume(session_id)
if orphan is None:
return
async def detach_orphan() -> None:
with suppress(DuplexSessionError):
await self._omni.detach_session(session_id, expected_lease_generation=orphan)
rollback = asyncio.ensure_future(detach_orphan())
rollback.add_done_callback(_log_rollback_failure)
await asyncio.shield(rollback)
async def _detach_lease(self, released: DuplexDetachedAttachment) -> None:
"""Engine-owned disconnect grace for the lease a dropped connection was serving."""
with suppress(DuplexSessionError):
await self._omni.detach_session(released.session_id, expected_lease_generation=released.lease_generation)
# ------------------------------------------------------------------ #
# Outbound pump (session-scoped, survives reconnects) #
# ------------------------------------------------------------------ #
@@ -420,9 +559,13 @@ class OmniDuplexSessionHandler:
if handle is None or handle.closed:
return
if handle.capabilities.supports_session_resume:
if await self._attachment_registry.detach(session_id, attachment_generation=None):
with suppress(DuplexSessionError):
await self._omni.detach_session(session_id)
# The lease generation comes with the dropped attachment, captured
# under the registry lock: the shared handle may already carry the
# generation of a resume that is waiting to activate behind this
# very send, and that lease is not ours to detach.
released = await self._attachment_registry.release_attachment(session_id, attachment_generation=None)
if released is not None:
await self._detach_lease(released)
return
with suppress(DuplexSessionError):
await handle.close(reason="disconnect")
@@ -532,10 +675,12 @@ class OmniDuplexSessionHandler:
if handle.closed:
return
if handle.capabilities.supports_session_resume:
if await self._attachment_registry.detach(handle.session_id, attachment_generation=attachment.generation):
released = await self._attachment_registry.release_attachment(
handle.session_id, attachment_generation=attachment.generation
)
if released is not None:
# Engine-owned disconnect grace: expiry arrives as session.expired.
with suppress(DuplexSessionError):
await self._omni.detach_session(handle.session_id)
await self._detach_lease(released)
# Otherwise a newer connection already took the session over; the
# replaced socket's disconnect must not touch it.
return
@@ -184,6 +184,15 @@ class DuplexSessionAttachmentCreated:
resume_token: ResumeToken = field(repr=False)
@dataclass(frozen=True)
class DuplexDetachedAttachment:
"""The transport a detach dropped, with the engine lease generation it was serving."""
session_id: str
attachment_generation: int
lease_generation: int
@dataclass(frozen=True)
class DuplexSessionResumeResult:
session_id: str
@@ -200,6 +209,21 @@ class _DuplexSessionAttachmentState:
journal: DuplexEventJournal
attachment: DuplexTransportAttachment | None
attachment_generation: int
#: The engine lease generation the attached connection is serving: the one
#: its open or resume produced, or one an abandoned resume handed to it.
#: Read together with the attachment under the lock, so a detach always
#: gives back the lease of the connection it drops, never a newer one.
lease_generation: int = 0
#: Resumes between ``begin_resume`` and ``end_resume``: their engine lease
#: resume is in flight or landed, and they have not activated yet. While
#: one is pending, the next activation is the connection that will serve,
#: so an orphaned generation waits for it instead of going to the socket
#: that is about to be replaced.
pending_resumes: int = 0
#: An engine lease generation nobody serves yet (its resume was abandoned,
#: or its provisional attachment was rolled back), waiting for the next
#: activation to absorb it. Generations only move forward.
orphaned_lease_generation: int | None = None
outbound_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
recovery_token_digest: bytes | None = field(default=None, repr=False)
@@ -231,6 +255,7 @@ class DuplexSessionAttachmentRegistry:
*,
send: Callable[[dict[str, object]], Awaitable[None]],
close: Callable[[str], Awaitable[None]],
lease_generation: int = 0,
) -> DuplexSessionAttachmentCreated:
async with self._lock:
if session_id in self._sessions:
@@ -251,6 +276,7 @@ class DuplexSessionAttachmentRegistry:
close=close,
),
attachment_generation=generation,
lease_generation=lease_generation,
)
return DuplexSessionAttachmentCreated(
session_id=session_id,
@@ -310,14 +336,91 @@ class DuplexSessionAttachmentRegistry:
session answers ``False`` so a second disconnect signal for the same
socket cannot restart the engine's disconnect grace window.
"""
return await self.release_attachment(session_id, attachment_generation=attachment_generation) is not None
async def release_attachment(
self, session_id: str, *, attachment_generation: int | None = None
) -> DuplexDetachedAttachment | None:
"""``detach`` that also hands back the lease generation the dropped connection was serving.
Captured under the same lock as the attachment: the shared session
handle may already carry a newer generation from a resume that has
not activated yet, and the engine detach that follows must be fenced
on the dropped connection's own lease, not on that one.
"""
async with self._lock:
state = self._sessions.get(session_id)
if state is None or state.attachment is None:
return False
return None
if attachment_generation is not None and state.attachment_generation != attachment_generation:
return False
return None
state.attachment = None
return True
return DuplexDetachedAttachment(
session_id=session_id,
attachment_generation=state.attachment_generation,
lease_generation=state.lease_generation,
)
async def begin_resume(self, session_id: str) -> None:
"""Claim that a resume of ``session_id`` is about to land and activate.
Held from before the engine lease resume until ``end_resume``. While
any claim is open, a lease generation that loses its owner is parked
for the next activation (see ``settle_lease_generation``) instead of
being handed to the currently attached connection, which that
activation is about to replace.
"""
async with self._lock:
self._require(session_id).pending_resumes += 1
async def end_resume(self, session_id: str) -> int | None:
"""Release a ``begin_resume`` claim, whether or not the resume activated.
Returns a lease generation the caller must detach itself: one that
was parked for an activation that never came, with nobody attached
to take it over. ``None`` when nothing is owed.
"""
async with self._lock:
state = self._sessions.get(session_id)
if state is None:
return None
state.pending_resumes = max(0, state.pending_resumes - 1)
return self._settle_orphan_locked(state)
async def settle_lease_generation(self, session_id: str, lease_generation: int) -> int | None:
"""Find an owner for the engine lease generation of a resume its connection never served.
The engine already bumped its lease to ``lease_generation``. The owner
is, in order: the resume waiting to activate (it will serve the session
and absorbs the generation when it activates); else the connection
attached right now (it keeps serving, and its own disconnect must be
able to detach this lease); else nobody, in which case the generation
is returned and the caller puts it back into disconnect grace.
Generations only move forward, so an older orphan never displaces a
newer one an owner already holds.
"""
async with self._lock:
state = self._sessions.get(session_id)
if state is None:
# The session is gone from the registry: the engine session
# closed with it, nothing is left to own.
return None
state.orphaned_lease_generation = max(state.orphaned_lease_generation or 0, lease_generation)
return self._settle_orphan_locked(state)
@staticmethod
def _settle_orphan_locked(state: _DuplexSessionAttachmentState) -> int | None:
orphan = state.orphaned_lease_generation
if orphan is None:
return None
if state.pending_resumes > 0:
# The next activation absorbs it (see ``resume``).
return None
state.orphaned_lease_generation = None
if state.attachment is not None:
state.lease_generation = max(state.lease_generation, orphan)
return None
return orphan
async def has_attachment(self, session_id: str) -> bool:
"""Whether some connection is attached right now, whoever it is.
@@ -362,6 +465,7 @@ class DuplexSessionAttachmentRegistry:
send: Callable[[dict[str, object]], Awaitable[None]],
close: Callable[[str], Awaitable[None]],
activation_payload_factory: Callable[[ResumeToken, int], Mapping[str, object]] | None = None,
lease_generation: int | None = None,
) -> DuplexSessionResumeResult:
async with self._lock:
state = self._require(session_id)
@@ -384,6 +488,15 @@ class DuplexSessionAttachmentRegistry:
send=send,
close=close,
)
# The activating connection serves the newest lease the engine
# holds: its own resume's generation, or a newer one whose
# resume was abandoned while this one was pending.
state.lease_generation = max(
state.lease_generation,
lease_generation if lease_generation is not None else 0,
state.orphaned_lease_generation or 0,
)
state.orphaned_lease_generation = None
if activation_payload_factory is not None:
try:
await send(dict(activation_payload_factory(rotated_token, attachment_generation)))
@@ -418,18 +531,26 @@ class DuplexSessionAttachmentRegistry:
*,
state: _DuplexSessionAttachmentState,
attachment_generation: int,
accepted_token_digest: bytes,
accepted_token_digest: bytes | None,
) -> None:
"""Undo a resume whose activation never reached the client.
Generation-checked: a later resume that already took over must keep its
attachment. Restoring ``recovery_token_digest`` is what lets the caller
retry with the token it presented, which ``resume`` had rotated past.
The lease the provisional attachment was serving loses its owner here.
That is not only its own resume's generation: a resume abandoned while
this one was activating may have handed it a newer one. It is parked as
the orphan, so the caller's settlement (which only knows its own
generation) and ``end_resume`` give back the newest lease, never a
stale one the engine would refuse.
"""
async with self._lock:
if self._sessions.get(session_id) is state and state.attachment_generation == attachment_generation:
state.attachment = None
state.recovery_token_digest = accepted_token_digest
state.orphaned_lease_generation = max(state.orphaned_lease_generation or 0, state.lease_generation)
async def close(self, session_id: str) -> DuplexTransportAttachment | None:
async with self._lock: