[Bugfix] Compensate cancelled duplex open, append and resume; fix serving README drift (#7636 Issues 1, 2, 3, 21) (#7785)

Signed-off-by: chickeyton <ngton2014@gmail.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-09-24 19:23:08 -07:00
1 parent 551a03102f
commit 9e7e3a802c
19 files changed
+1697 -96

No files matched your search

+59 -11
View File
@@ -13,6 +13,13 @@ That compensation is the reason this is an object. The five steps of an append
all need the same eleven values, so as closures over ``_start_append`` they read
as one function with five entry points and no way to test any of them. Named
fields make the captured state explicit and the compensation a method.
The precreated response has two other owners the compensation defers to. When
the runner cancels the append on purpose (``DuplexSessionTasks.cancel_append_tasks``,
for a ``response.cancel``, a barge-in or a close), the runner ends that response
itself right after, with the cancel status the client asked for. When the
session is already closing, the close ends it. Everywhere else, the append is
the last thing that knows the response will never be filled, and it fails it.
"""
from __future__ import annotations
@@ -53,6 +60,10 @@ class AppendAttempt:
retained_committed_payload: dict[str, object] | None
#: Response reserved before submission, to be failed if submission does not happen.
precreated_response_id: str | None
#: Whether ``_start_append`` bound the session's active request to this
#: append (a final or response-bound one), so an append that never happens
#: has to unbind it again.
owns_request: bool = False
#: Commits timing state once the runtime accepts the append (before any
#: returned output event can clear the continuation chain).
on_append_accepted: Callable[[float], None] | None = None
@@ -75,11 +86,31 @@ class AppendAttempt:
):
self.ctx.session.release_input_bytes(model_state.clear_committed_audio())
def abandon(self) -> None:
"""Give back everything this append reserved but never used."""
def abandon(self, *, reason: str = "runtime_append_failed", response_taken_over: bool = False) -> None:
"""Give back everything this append reserved but never used.
That includes the response it precreated: once the append is off,
nothing will ever fill it, and a client that saw ``response.created``
is owed a ``response.done``. It is left alone only when somebody else
ends it: the runner that cancelled this append on purpose
(``response_taken_over``) or the close already under way.
"""
if self.pcm_reservation is not None:
self.pcm_reservation.rollback()
self.discard_retained_audio()
if response_taken_over:
return
session = self.ctx.session
if self.ctx.run.closing or session.state != DuplexSessionState.OPEN:
# The close ends the active response and request itself, with the
# session's one terminal event after them.
return
if self.owns_request and session.epoch == self.epoch:
# Compare-before-clear: only if the session still points at this
# append's request, exactly as the submitted path does.
session.clear_request(self.request_id)
if self._precreated_response_still_active():
self._fail_precreated_response(reason)
def release_on_failure(self, done: asyncio.Task[bool]) -> None:
"""Done-callback: a cancelled or failed append must not hold the audio."""
@@ -105,7 +136,22 @@ class AppendAttempt:
# ------------------------------------------------------------------ #
async def run_in_wire_order(self, predecessor: asyncio.Task[bool] | None) -> bool:
"""Wait for the previous append, then submit this one if it still applies."""
"""Wait for the previous append, then submit this one if it still applies.
A cancellation is compensated here, wherever it lands (waiting for the
predecessor or mid-submit): the reservations go back, and the
precreated response is failed unless the runner cancelled the append
itself, in which case it ends the response with the status the
cancellation was for.
"""
try:
return await self._run(predecessor)
except asyncio.CancelledError:
taken_over = self.ctx.tasks.cancelled_by_runner(asyncio.current_task())
self.abandon(reason="append_cancelled", response_taken_over=taken_over)
raise
async def _run(self, predecessor: asyncio.Task[bool] | None) -> bool:
if predecessor is not None:
try:
predecessor_ok = await predecessor
@@ -127,7 +173,11 @@ class AppendAttempt:
# Called off, not failed: the chain behind it still runs.
return True
if self.pcm_reservation is not None and not self.pcm_reservation.active:
self.discard_retained_audio()
# The client cleared the input buffer while this append waited its
# turn: the audio is gone, so is the response it was going to
# answer, and that is the client's doing, not a runtime failure.
# The reservation rollback itself is already a no-op.
self.abandon(reason="input_cleared")
return False
return await self._submit()
@@ -152,8 +202,6 @@ class AppendAttempt:
self.discard_retained_audio()
else:
self.abandon()
if not append_ok and self._precreated_response_still_active():
self._fail_precreated_response()
if not append_ok and session.state == DuplexSessionState.CLOSED:
self.ctx.run.runtime_closed = True
return False
@@ -171,11 +219,11 @@ class AppendAttempt:
self.out.emit_events([session.signal_turn(DuplexTurnEventType.USER_STARTED.value)])
return append_ok
except asyncio.CancelledError:
if self.pcm_reservation is not None:
self.pcm_reservation.rollback()
raise
except Exception as exc:
self.abandon()
# Compensate while the session is still open, so the client gets
# the failed ``response.done`` before the error and the close.
self.abandon(reason="runtime_append_task_failed")
logger.exception("Native duplex append task failed: %s", exc)
self.model.send_runtime_error("runtime_append_task_failed", exc)
if session.state != DuplexSessionState.CLOSED:
@@ -188,7 +236,7 @@ class AppendAttempt:
and self.ctx.session.active_response_id == self.precreated_response_id
)
def _fail_precreated_response(self) -> None:
def _fail_precreated_response(self, reason: str) -> None:
session = self.ctx.session
session.end_response(commit_text=False)
self.out.emit(
@@ -199,7 +247,7 @@ class AppendAttempt:
"epoch": session.epoch,
"committed": False,
"status": "failed",
"status_details": {"type": "failed", "reason": "runtime_append_failed"},
"status_details": {"type": "failed", "reason": reason},
"playback": session.playback.as_dict(),
}
)
+21 -2
View File
@@ -20,7 +20,7 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Protocol, TypeVar
from typing import TYPE_CHECKING, Any, Protocol, TypeVar
if TYPE_CHECKING:
from vllm.outputs import RequestOutput
@@ -117,6 +117,10 @@ class DuplexSessionTasks:
append_tasks: dict[asyncio.Task[bool], DuplexAppendTaskMeta] = field(default_factory=dict)
append_tail: asyncio.Task[bool] | None = None
active_response_task: asyncio.Task[None] | None = None
#: Appends the runner cancelled on purpose (a cancel, barge-in or close).
#: The runner ends the response such an append precreated, so the append
#: leaves it alone on its way out instead of failing it.
runner_cancelled: set[asyncio.Task[bool]] = field(default_factory=set)
def track_append_task(
self,
@@ -127,16 +131,31 @@ class DuplexSessionTasks:
response_bound: bool,
) -> None:
self.append_tasks[task] = DuplexAppendTaskMeta(epoch, final, response_bound)
task.add_done_callback(self.append_tasks.pop)
task.add_done_callback(self._forget_append_task)
def _forget_append_task(self, task: asyncio.Task[bool]) -> None:
self.append_tasks.pop(task, None)
self.runner_cancelled.discard(task)
def cancelled_by_runner(self, task: asyncio.Task[Any] | None) -> bool:
"""Whether ``task`` is an append the runner cancelled through ``cancel_append_tasks``."""
return task is not None and task in self.runner_cancelled
def has_response_bound_append_tasks(self) -> bool:
return any(meta.response_bound for meta in self.append_tasks.values())
async def cancel_append_tasks(self, timeout_s: float = 0.25, *, response_bound_only: bool = False) -> bool:
"""Cancel the tracked appends; the caller owns their responses from here on.
Every caller ends the active response itself right after (with the
cancel status, or silently as part of a close), so the cancelled
appends are told not to fail the response they precreated.
"""
tasks = [task for task, meta in self.append_tasks.items() if not response_bound_only or meta.response_bound]
if not tasks:
return False
cancelled_tail = self.append_tail if self.append_tail in tasks else None
self.runner_cancelled.update(tasks)
for task in tasks:
task.cancel()
try:
@@ -280,9 +280,9 @@ class DuplexEngineSession:
self._validate_fence(self.fence)
self.lease.detach(self._clock())
def resume_lease(self, *, expected_lease_generation: int) -> int:
def resume_lease(self, *, expected_lease_generation: int, control_id: str | None = None) -> int:
self._validate_fence(self.fence)
return self.lease.resume(self._clock(), expected_generation=expected_lease_generation)
return self.lease.resume(self._clock(), expected_generation=expected_lease_generation, control_id=control_id)
def begin_lease_operation(self, fence: DuplexFence, operation_id: str) -> None:
self._validate_fence(fence)
+13 -1
View File
@@ -40,6 +40,12 @@ class DuplexLeaseState:
detached_at: float | None = None
active_operations: set[str] = field(default_factory=set)
terminal_reason: str | None = None
#: Control id of the resume that produced ``generation`` (``None`` for the
#: open). A resume replayed with the same id is answered with the current
#: generation instead of bumping it again: the caller lost the first
#: answer (its RPC waiter was cancelled or timed out) and is finding out
#: whether the resume landed.
resumed_by: str | None = None
@property
def expires_at(self) -> float | None:
@@ -61,11 +67,16 @@ class DuplexLeaseState:
self.touch(now, DuplexLeaseActivity.DETACH)
self.detached_at = now
def resume(self, now: float, *, expected_generation: int) -> int:
def resume(self, now: float, *, expected_generation: int, control_id: str | None = None) -> int:
self._require_open()
if control_id is not None and control_id == self.resumed_by:
# The same resume asking again: it already produced this
# generation, so answer without changing anything.
return self.generation
if expected_generation != self.generation:
raise ValueError(f"duplex lease generation mismatch: expected {self.generation}, got {expected_generation}")
self.generation += 1
self.resumed_by = control_id
self.detached_at = None
self.last_activity = now
return self.generation
@@ -104,6 +115,7 @@ class DuplexLeaseState:
return False
self.terminal_reason = reason
self.generation += 1
self.resumed_by = None
self.detached_at = None
self.active_operations.clear()
return True
+71 -15
View File
@@ -523,32 +523,74 @@ class DuplexSessionManager:
# and the stage resource straight back.
self._abandoned_opens.discard(session_id)
await self._close_runner(runner, kind="close", reason="open_abandoned")
except asyncio.CancelledError:
# A cancel (engine teardown, task cancellation) while awaiting the
# plugin or the result sink is not an ``Exception``: without this
# branch the runner stayed in ``runners`` with its Stage0
# reservation, and since admission counts runners, every cancelled
# open burned one slot for good. Nobody is answered: the caller's
# RPC waiter is being torn down with us or times out.
logger.info("open_duplex_session cancelled for %s; rolling the admission back", session_id)
await self._rollback_open(session_id, session=session, runner=runner)
raise
except Exception as exc:
error_code, _, _ = self._control_error(exc)
if error_code in {"resource_exhausted", "session_exists"}:
logger.info("open_duplex_session rejected: %s", exc)
else:
logger.exception("open_duplex_session failed: %s", exc)
if runner is not None and self.runners.get(session_id) is runner:
self.runners.pop(session_id, None)
try:
await runner.shutdown()
except Exception:
logger.exception("duplex open rollback: runner shutdown failed for %s", session_id)
if session is not None:
reserved = session.release_all_requests()
if reserved:
try:
await self.stage_port.cleanup(list(reserved))
except Exception:
logger.warning("duplex open rollback: request cleanup pending for %s", session_id)
self._unregister_session_requests(session_id)
await self._rollback_open(session_id, session=session, runner=runner)
await self._put_result(message, operation="open", ok=False, error=exc)
finally:
if holds_admission_slot:
self._admitting.discard(session_id)
self._abandoned_opens.discard(session_id)
async def _rollback_open(
self,
session_id: str,
*,
session: DuplexEngineSession | None,
runner: DuplexSessionRunner | None,
) -> None:
"""Undo whatever an open that will not complete already did.
The bookkeeping that decides admission (``runners`` and the session's
request reservations) is undone synchronously first, so a second
cancellation landing in the awaits below cannot leave the slot burned.
The Stage0 request ids are recorded as a pending request cleanup
before anything is awaited: if the runner shutdown or the cleanup
itself is cancelled or fails, the reaper retries the cleanup instead
of the orchestrator keeping the request state forever.
"""
owned_runner = runner is not None and self.runners.get(session_id) is runner
if owned_runner:
self.runners.pop(session_id, None)
pending: _PendingRequestCleanup | None = None
key: tuple[str, int] | None = None
if session is not None:
reserved = tuple(session.release_all_requests())
self._unregister_session_requests(session_id)
if reserved:
key = (session_id, session.lease_generation)
pending = _PendingRequestCleanup(
session_id=session_id,
lease_generation=session.lease_generation,
request_ids=reserved,
abort=False,
)
self._pending_request_cleanups[key] = pending
if owned_runner and runner is not None:
try:
await runner.shutdown()
except Exception:
logger.exception("duplex open rollback: runner shutdown failed for %s", session_id)
if pending is not None and key is not None:
try:
await self._complete_request_cleanup(key, pending)
except Exception:
logger.warning("duplex open rollback: request cleanup pending for %s; the reaper retries", session_id)
def _require_runner(self, session_id: str) -> DuplexSessionRunner:
runner = self.runners.get(session_id)
if runner is None:
@@ -668,7 +710,13 @@ class DuplexSessionManager:
runner = self._require_runner(message.session_id)
session = runner.session
try:
session.resume_lease(expected_lease_generation=message.expected_lease_generation)
# Keyed by the control id, so a caller that lost the answer
# (waiter cancelled or timed out) can replay the same resume
# to learn whether it landed instead of resuming twice.
session.resume_lease(
expected_lease_generation=message.expected_lease_generation,
control_id=message.control_id,
)
except ValueError as exc:
raise DuplexSessionError(str(exc), code="session_resume_conflict") from exc
await self._put_result(message, operation="resume", ok=True, session=session)
@@ -683,6 +731,14 @@ class DuplexSessionManager:
session = runner.session
activity = DuplexLeaseActivity(message.activity)
if activity is DuplexLeaseActivity.DETACH:
expected = message.expected_lease_generation
if expected is not None and expected != session.lease_generation:
# The caller is giving up a lease it no longer holds: a
# later resume owns the current one and must keep it.
raise DuplexSessionError(
f"duplex lease generation mismatch: expected {session.lease_generation}, got {expected}",
code="session_resume_conflict",
)
session.detach_lease()
else:
session.touch_lease(activity)
@@ -1107,6 +1107,7 @@ class DuplexSessionRunner:
operation_id=operation_id,
retained_committed_payload=retained_committed_payload,
precreated_response_id=session.active_response_id if precreate_response else None,
owns_request=final or precreate_response,
on_append_accepted=on_append_accepted,
before_append=before_append,
)