[Bugfix] Compensate cancelled duplex open, append and resume; fix serving README drift (#7636 Issues 1, 2, 3, 21) (#7785)
Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
19 files changed
+1697
-96
No files matched your search
@@ -13,6 +13,13 @@ That compensation is the reason this is an object. The five steps of an append
|
||||
all need the same eleven values, so as closures over ``_start_append`` they read
|
||||
as one function with five entry points and no way to test any of them. Named
|
||||
fields make the captured state explicit and the compensation a method.
|
||||
|
||||
The precreated response has two other owners the compensation defers to. When
|
||||
the runner cancels the append on purpose (``DuplexSessionTasks.cancel_append_tasks``,
|
||||
for a ``response.cancel``, a barge-in or a close), the runner ends that response
|
||||
itself right after, with the cancel status the client asked for. When the
|
||||
session is already closing, the close ends it. Everywhere else, the append is
|
||||
the last thing that knows the response will never be filled, and it fails it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -53,6 +60,10 @@ class AppendAttempt:
|
||||
retained_committed_payload: dict[str, object] | None
|
||||
#: Response reserved before submission, to be failed if submission does not happen.
|
||||
precreated_response_id: str | None
|
||||
#: Whether ``_start_append`` bound the session's active request to this
|
||||
#: append (a final or response-bound one), so an append that never happens
|
||||
#: has to unbind it again.
|
||||
owns_request: bool = False
|
||||
#: Commits timing state once the runtime accepts the append (before any
|
||||
#: returned output event can clear the continuation chain).
|
||||
on_append_accepted: Callable[[float], None] | None = None
|
||||
@@ -75,11 +86,31 @@ class AppendAttempt:
|
||||
):
|
||||
self.ctx.session.release_input_bytes(model_state.clear_committed_audio())
|
||||
|
||||
def abandon(self) -> None:
|
||||
"""Give back everything this append reserved but never used."""
|
||||
def abandon(self, *, reason: str = "runtime_append_failed", response_taken_over: bool = False) -> None:
|
||||
"""Give back everything this append reserved but never used.
|
||||
|
||||
That includes the response it precreated: once the append is off,
|
||||
nothing will ever fill it, and a client that saw ``response.created``
|
||||
is owed a ``response.done``. It is left alone only when somebody else
|
||||
ends it: the runner that cancelled this append on purpose
|
||||
(``response_taken_over``) or the close already under way.
|
||||
"""
|
||||
if self.pcm_reservation is not None:
|
||||
self.pcm_reservation.rollback()
|
||||
self.discard_retained_audio()
|
||||
if response_taken_over:
|
||||
return
|
||||
session = self.ctx.session
|
||||
if self.ctx.run.closing or session.state != DuplexSessionState.OPEN:
|
||||
# The close ends the active response and request itself, with the
|
||||
# session's one terminal event after them.
|
||||
return
|
||||
if self.owns_request and session.epoch == self.epoch:
|
||||
# Compare-before-clear: only if the session still points at this
|
||||
# append's request, exactly as the submitted path does.
|
||||
session.clear_request(self.request_id)
|
||||
if self._precreated_response_still_active():
|
||||
self._fail_precreated_response(reason)
|
||||
|
||||
def release_on_failure(self, done: asyncio.Task[bool]) -> None:
|
||||
"""Done-callback: a cancelled or failed append must not hold the audio."""
|
||||
@@ -105,7 +136,22 @@ class AppendAttempt:
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
async def run_in_wire_order(self, predecessor: asyncio.Task[bool] | None) -> bool:
|
||||
"""Wait for the previous append, then submit this one if it still applies."""
|
||||
"""Wait for the previous append, then submit this one if it still applies.
|
||||
|
||||
A cancellation is compensated here, wherever it lands (waiting for the
|
||||
predecessor or mid-submit): the reservations go back, and the
|
||||
precreated response is failed unless the runner cancelled the append
|
||||
itself, in which case it ends the response with the status the
|
||||
cancellation was for.
|
||||
"""
|
||||
try:
|
||||
return await self._run(predecessor)
|
||||
except asyncio.CancelledError:
|
||||
taken_over = self.ctx.tasks.cancelled_by_runner(asyncio.current_task())
|
||||
self.abandon(reason="append_cancelled", response_taken_over=taken_over)
|
||||
raise
|
||||
|
||||
async def _run(self, predecessor: asyncio.Task[bool] | None) -> bool:
|
||||
if predecessor is not None:
|
||||
try:
|
||||
predecessor_ok = await predecessor
|
||||
@@ -127,7 +173,11 @@ class AppendAttempt:
|
||||
# Called off, not failed: the chain behind it still runs.
|
||||
return True
|
||||
if self.pcm_reservation is not None and not self.pcm_reservation.active:
|
||||
self.discard_retained_audio()
|
||||
# The client cleared the input buffer while this append waited its
|
||||
# turn: the audio is gone, so is the response it was going to
|
||||
# answer, and that is the client's doing, not a runtime failure.
|
||||
# The reservation rollback itself is already a no-op.
|
||||
self.abandon(reason="input_cleared")
|
||||
return False
|
||||
return await self._submit()
|
||||
|
||||
@@ -152,8 +202,6 @@ class AppendAttempt:
|
||||
self.discard_retained_audio()
|
||||
else:
|
||||
self.abandon()
|
||||
if not append_ok and self._precreated_response_still_active():
|
||||
self._fail_precreated_response()
|
||||
if not append_ok and session.state == DuplexSessionState.CLOSED:
|
||||
self.ctx.run.runtime_closed = True
|
||||
return False
|
||||
@@ -171,11 +219,11 @@ class AppendAttempt:
|
||||
self.out.emit_events([session.signal_turn(DuplexTurnEventType.USER_STARTED.value)])
|
||||
return append_ok
|
||||
except asyncio.CancelledError:
|
||||
if self.pcm_reservation is not None:
|
||||
self.pcm_reservation.rollback()
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.abandon()
|
||||
# Compensate while the session is still open, so the client gets
|
||||
# the failed ``response.done`` before the error and the close.
|
||||
self.abandon(reason="runtime_append_task_failed")
|
||||
logger.exception("Native duplex append task failed: %s", exc)
|
||||
self.model.send_runtime_error("runtime_append_task_failed", exc)
|
||||
if session.state != DuplexSessionState.CLOSED:
|
||||
@@ -188,7 +236,7 @@ class AppendAttempt:
|
||||
and self.ctx.session.active_response_id == self.precreated_response_id
|
||||
)
|
||||
|
||||
def _fail_precreated_response(self) -> None:
|
||||
def _fail_precreated_response(self, reason: str) -> None:
|
||||
session = self.ctx.session
|
||||
session.end_response(commit_text=False)
|
||||
self.out.emit(
|
||||
@@ -199,7 +247,7 @@ class AppendAttempt:
|
||||
"epoch": session.epoch,
|
||||
"committed": False,
|
||||
"status": "failed",
|
||||
"status_details": {"type": "failed", "reason": "runtime_append_failed"},
|
||||
"status_details": {"type": "failed", "reason": reason},
|
||||
"playback": session.playback.as_dict(),
|
||||
}
|
||||
)
|
||||
@@ -20,7 +20,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol, TypeVar
|
||||
from typing import TYPE_CHECKING, Any, Protocol, TypeVar
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.outputs import RequestOutput
|
||||
@@ -117,6 +117,10 @@ class DuplexSessionTasks:
|
||||
append_tasks: dict[asyncio.Task[bool], DuplexAppendTaskMeta] = field(default_factory=dict)
|
||||
append_tail: asyncio.Task[bool] | None = None
|
||||
active_response_task: asyncio.Task[None] | None = None
|
||||
#: Appends the runner cancelled on purpose (a cancel, barge-in or close).
|
||||
#: The runner ends the response such an append precreated, so the append
|
||||
#: leaves it alone on its way out instead of failing it.
|
||||
runner_cancelled: set[asyncio.Task[bool]] = field(default_factory=set)
|
||||
|
||||
def track_append_task(
|
||||
self,
|
||||
@@ -127,16 +131,31 @@ class DuplexSessionTasks:
|
||||
response_bound: bool,
|
||||
) -> None:
|
||||
self.append_tasks[task] = DuplexAppendTaskMeta(epoch, final, response_bound)
|
||||
task.add_done_callback(self.append_tasks.pop)
|
||||
task.add_done_callback(self._forget_append_task)
|
||||
|
||||
def _forget_append_task(self, task: asyncio.Task[bool]) -> None:
|
||||
self.append_tasks.pop(task, None)
|
||||
self.runner_cancelled.discard(task)
|
||||
|
||||
def cancelled_by_runner(self, task: asyncio.Task[Any] | None) -> bool:
|
||||
"""Whether ``task`` is an append the runner cancelled through ``cancel_append_tasks``."""
|
||||
return task is not None and task in self.runner_cancelled
|
||||
|
||||
def has_response_bound_append_tasks(self) -> bool:
|
||||
return any(meta.response_bound for meta in self.append_tasks.values())
|
||||
|
||||
async def cancel_append_tasks(self, timeout_s: float = 0.25, *, response_bound_only: bool = False) -> bool:
|
||||
"""Cancel the tracked appends; the caller owns their responses from here on.
|
||||
|
||||
Every caller ends the active response itself right after (with the
|
||||
cancel status, or silently as part of a close), so the cancelled
|
||||
appends are told not to fail the response they precreated.
|
||||
"""
|
||||
tasks = [task for task, meta in self.append_tasks.items() if not response_bound_only or meta.response_bound]
|
||||
if not tasks:
|
||||
return False
|
||||
cancelled_tail = self.append_tail if self.append_tail in tasks else None
|
||||
self.runner_cancelled.update(tasks)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
try:
|
||||
|
||||
@@ -280,9 +280,9 @@ class DuplexEngineSession:
|
||||
self._validate_fence(self.fence)
|
||||
self.lease.detach(self._clock())
|
||||
|
||||
def resume_lease(self, *, expected_lease_generation: int) -> int:
|
||||
def resume_lease(self, *, expected_lease_generation: int, control_id: str | None = None) -> int:
|
||||
self._validate_fence(self.fence)
|
||||
return self.lease.resume(self._clock(), expected_generation=expected_lease_generation)
|
||||
return self.lease.resume(self._clock(), expected_generation=expected_lease_generation, control_id=control_id)
|
||||
|
||||
def begin_lease_operation(self, fence: DuplexFence, operation_id: str) -> None:
|
||||
self._validate_fence(fence)
|
||||
|
||||
@@ -40,6 +40,12 @@ class DuplexLeaseState:
|
||||
detached_at: float | None = None
|
||||
active_operations: set[str] = field(default_factory=set)
|
||||
terminal_reason: str | None = None
|
||||
#: Control id of the resume that produced ``generation`` (``None`` for the
|
||||
#: open). A resume replayed with the same id is answered with the current
|
||||
#: generation instead of bumping it again: the caller lost the first
|
||||
#: answer (its RPC waiter was cancelled or timed out) and is finding out
|
||||
#: whether the resume landed.
|
||||
resumed_by: str | None = None
|
||||
|
||||
@property
|
||||
def expires_at(self) -> float | None:
|
||||
@@ -61,11 +67,16 @@ class DuplexLeaseState:
|
||||
self.touch(now, DuplexLeaseActivity.DETACH)
|
||||
self.detached_at = now
|
||||
|
||||
def resume(self, now: float, *, expected_generation: int) -> int:
|
||||
def resume(self, now: float, *, expected_generation: int, control_id: str | None = None) -> int:
|
||||
self._require_open()
|
||||
if control_id is not None and control_id == self.resumed_by:
|
||||
# The same resume asking again: it already produced this
|
||||
# generation, so answer without changing anything.
|
||||
return self.generation
|
||||
if expected_generation != self.generation:
|
||||
raise ValueError(f"duplex lease generation mismatch: expected {self.generation}, got {expected_generation}")
|
||||
self.generation += 1
|
||||
self.resumed_by = control_id
|
||||
self.detached_at = None
|
||||
self.last_activity = now
|
||||
return self.generation
|
||||
@@ -104,6 +115,7 @@ class DuplexLeaseState:
|
||||
return False
|
||||
self.terminal_reason = reason
|
||||
self.generation += 1
|
||||
self.resumed_by = None
|
||||
self.detached_at = None
|
||||
self.active_operations.clear()
|
||||
return True
|
||||
|
||||
@@ -523,32 +523,74 @@ class DuplexSessionManager:
|
||||
# and the stage resource straight back.
|
||||
self._abandoned_opens.discard(session_id)
|
||||
await self._close_runner(runner, kind="close", reason="open_abandoned")
|
||||
except asyncio.CancelledError:
|
||||
# A cancel (engine teardown, task cancellation) while awaiting the
|
||||
# plugin or the result sink is not an ``Exception``: without this
|
||||
# branch the runner stayed in ``runners`` with its Stage0
|
||||
# reservation, and since admission counts runners, every cancelled
|
||||
# open burned one slot for good. Nobody is answered: the caller's
|
||||
# RPC waiter is being torn down with us or times out.
|
||||
logger.info("open_duplex_session cancelled for %s; rolling the admission back", session_id)
|
||||
await self._rollback_open(session_id, session=session, runner=runner)
|
||||
raise
|
||||
except Exception as exc:
|
||||
error_code, _, _ = self._control_error(exc)
|
||||
if error_code in {"resource_exhausted", "session_exists"}:
|
||||
logger.info("open_duplex_session rejected: %s", exc)
|
||||
else:
|
||||
logger.exception("open_duplex_session failed: %s", exc)
|
||||
if runner is not None and self.runners.get(session_id) is runner:
|
||||
self.runners.pop(session_id, None)
|
||||
try:
|
||||
await runner.shutdown()
|
||||
except Exception:
|
||||
logger.exception("duplex open rollback: runner shutdown failed for %s", session_id)
|
||||
if session is not None:
|
||||
reserved = session.release_all_requests()
|
||||
if reserved:
|
||||
try:
|
||||
await self.stage_port.cleanup(list(reserved))
|
||||
except Exception:
|
||||
logger.warning("duplex open rollback: request cleanup pending for %s", session_id)
|
||||
self._unregister_session_requests(session_id)
|
||||
await self._rollback_open(session_id, session=session, runner=runner)
|
||||
await self._put_result(message, operation="open", ok=False, error=exc)
|
||||
finally:
|
||||
if holds_admission_slot:
|
||||
self._admitting.discard(session_id)
|
||||
self._abandoned_opens.discard(session_id)
|
||||
|
||||
async def _rollback_open(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
session: DuplexEngineSession | None,
|
||||
runner: DuplexSessionRunner | None,
|
||||
) -> None:
|
||||
"""Undo whatever an open that will not complete already did.
|
||||
|
||||
The bookkeeping that decides admission (``runners`` and the session's
|
||||
request reservations) is undone synchronously first, so a second
|
||||
cancellation landing in the awaits below cannot leave the slot burned.
|
||||
The Stage0 request ids are recorded as a pending request cleanup
|
||||
before anything is awaited: if the runner shutdown or the cleanup
|
||||
itself is cancelled or fails, the reaper retries the cleanup instead
|
||||
of the orchestrator keeping the request state forever.
|
||||
"""
|
||||
owned_runner = runner is not None and self.runners.get(session_id) is runner
|
||||
if owned_runner:
|
||||
self.runners.pop(session_id, None)
|
||||
pending: _PendingRequestCleanup | None = None
|
||||
key: tuple[str, int] | None = None
|
||||
if session is not None:
|
||||
reserved = tuple(session.release_all_requests())
|
||||
self._unregister_session_requests(session_id)
|
||||
if reserved:
|
||||
key = (session_id, session.lease_generation)
|
||||
pending = _PendingRequestCleanup(
|
||||
session_id=session_id,
|
||||
lease_generation=session.lease_generation,
|
||||
request_ids=reserved,
|
||||
abort=False,
|
||||
)
|
||||
self._pending_request_cleanups[key] = pending
|
||||
if owned_runner and runner is not None:
|
||||
try:
|
||||
await runner.shutdown()
|
||||
except Exception:
|
||||
logger.exception("duplex open rollback: runner shutdown failed for %s", session_id)
|
||||
if pending is not None and key is not None:
|
||||
try:
|
||||
await self._complete_request_cleanup(key, pending)
|
||||
except Exception:
|
||||
logger.warning("duplex open rollback: request cleanup pending for %s; the reaper retries", session_id)
|
||||
|
||||
def _require_runner(self, session_id: str) -> DuplexSessionRunner:
|
||||
runner = self.runners.get(session_id)
|
||||
if runner is None:
|
||||
@@ -668,7 +710,13 @@ class DuplexSessionManager:
|
||||
runner = self._require_runner(message.session_id)
|
||||
session = runner.session
|
||||
try:
|
||||
session.resume_lease(expected_lease_generation=message.expected_lease_generation)
|
||||
# Keyed by the control id, so a caller that lost the answer
|
||||
# (waiter cancelled or timed out) can replay the same resume
|
||||
# to learn whether it landed instead of resuming twice.
|
||||
session.resume_lease(
|
||||
expected_lease_generation=message.expected_lease_generation,
|
||||
control_id=message.control_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise DuplexSessionError(str(exc), code="session_resume_conflict") from exc
|
||||
await self._put_result(message, operation="resume", ok=True, session=session)
|
||||
@@ -683,6 +731,14 @@ class DuplexSessionManager:
|
||||
session = runner.session
|
||||
activity = DuplexLeaseActivity(message.activity)
|
||||
if activity is DuplexLeaseActivity.DETACH:
|
||||
expected = message.expected_lease_generation
|
||||
if expected is not None and expected != session.lease_generation:
|
||||
# The caller is giving up a lease it no longer holds: a
|
||||
# later resume owns the current one and must keep it.
|
||||
raise DuplexSessionError(
|
||||
f"duplex lease generation mismatch: expected {session.lease_generation}, got {expected}",
|
||||
code="session_resume_conflict",
|
||||
)
|
||||
session.detach_lease()
|
||||
else:
|
||||
session.touch_lease(activity)
|
||||
|
||||
@@ -1107,6 +1107,7 @@ class DuplexSessionRunner:
|
||||
operation_id=operation_id,
|
||||
retained_committed_payload=retained_committed_payload,
|
||||
precreated_response_id=session.active_response_id if precreate_response else None,
|
||||
owns_request=final or precreate_response,
|
||||
on_append_accepted=on_append_accepted,
|
||||
before_append=before_append,
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user