[Bugfix] Fix colocate-async sleep admission race and deliver AR abort tokens (#6367)

Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com>
Signed-off-by: Cursor <cursoragent@cursor.com>
Signed-off-by: Samit <285365963@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
authored and GitHub committed 2026-08-29 09:11:39 +00:00
1 parent 5c47fd0bc8
commit bb5bf8fd0a
23 files changed
+1229 -157

No files matched your search

+7 -1
View File
@@ -67,6 +67,7 @@ from vllm_omni.engine.messages import (
EngineQueueMessage,
ErrorMessage,
InteractionMessage,
OutputMessage,
StageSubmissionMessage,
)
from vllm_omni.engine.orchestrator import Orchestrator
@@ -1920,12 +1921,16 @@ class AsyncOmniEngine:
self,
request_ids: list[str],
timeout: float | None = None,
) -> None:
) -> list[OutputMessage]:
"""Abort requests and wait for orchestrator acknowledgment.
Unlike :meth:`abort`, this generates an ``rpc_id``, correlates the
:class:`AbortResultMessage` via :class:`CorrelatedRpcClient`, and
raises if the orchestrator reports failure or times out.
Returns:
Final-stage AR abort ``OutputMessage`` list carrying partial
tokens generated before abort (empty for diffusion / no OP state).
"""
if not request_ids or getattr(self, "_shutdown_called", False):
return
@@ -1959,6 +1964,7 @@ class AsyncOmniEngine:
raise
if not result_msg.success:
raise RuntimeError(result_msg.error or "abort failed")
return list(result_msg.abort_outputs or [])
def submit_interaction(
self,
+14 -11
View File
@@ -48,17 +48,6 @@ class AbortRequestMessage(EngineQueueMessage, kw_only=True):
rpc_id: str | None = None
class AbortResultMessage(EngineQueueMessage, kw_only=True):
type: Literal["abort_result"] = "abort_result"
rpc_id: str
success: bool
error: str | None = None
@property
def rpc_correlation_key(self) -> tuple[str, str]:
return ("abort", self.rpc_id)
class InteractionMessage(EngineQueueMessage, kw_only=True):
type: Literal["interaction"] = "interaction"
request_id: str
@@ -113,6 +102,20 @@ class OutputMessage(EngineQueueMessage, kw_only=True):
stage_submit_ts: float | None = None
class AbortResultMessage(EngineQueueMessage, kw_only=True):
type: Literal["abort_result"] = "abort_result"
rpc_id: str
success: bool
error: str | None = None
# Final-stage AR abort outputs (partial tokens) for frontend delivery.
# Empty for diffusion / requests with no output-processor state.
abort_outputs: list[OutputMessage] | None = None
@property
def rpc_correlation_key(self) -> tuple[str, str]:
return ("abort", self.rpc_id)
class StageMetricsMessage(EngineQueueMessage, kw_only=True):
type: Literal["stage_metrics"] = "stage_metrics"
request_id: str
+63 -8
View File
@@ -871,14 +871,19 @@ class Orchestrator:
When ``msg.rpc_id`` is set, emit :class:`AbortResultMessage` after
stage aborts, binding release, and request cleanup complete so the
caller can await acknowledgment. Fire-and-forget aborts omit rpc_id.
AR final-stage abort outputs (partial tokens) are attached to the
result message when ``rpc_id`` is set, or enqueued on
``output_async_queue`` for fire-and-forget aborts.
"""
request_ids = msg.request_ids
error: str | None = None
abort_outputs: list[OutputMessage] = []
try:
# _cleanup_request_ids is CFG-aware: it expands aborted parents to
# their companions and fails a deferred parent whose companion is
# aborted before its output arrived.
await self._cleanup_request_ids(list(request_ids), abort=True)
abort_outputs = await self._cleanup_request_ids(list(request_ids), abort=True)
logger.info("[Orchestrator] Aborted request(s) %s", request_ids)
except Exception as exc:
error = str(exc)
@@ -893,8 +898,12 @@ class Orchestrator:
rpc_id=msg.rpc_id,
success=error is None,
error=error,
abort_outputs=abort_outputs or None,
)
)
elif abort_outputs:
for output_msg in abort_outputs:
await self.output_async_queue.put(output_msg)
async def _handle_interaction(self, msg: InteractionMessage) -> None:
"""Handle a midway interaction for an active streaming diffusion request."""
@@ -934,13 +943,53 @@ class Orchestrator:
)
)
async def _abort_request_ids(self, request_ids: list[str]) -> None:
"""Forward abort requests to all stage pools."""
async def _abort_request_ids(self, request_ids: list[str]) -> list[OutputMessage]:
"""Forward abort requests to all stage pools.
Collects final-stage AR abort outputs (partial tokens) while request
state is still available for stage-id filtering. Diffusion pools do
not contribute abort outputs.
"""
if not request_ids:
return
return []
abort_outputs: list[OutputMessage] = []
for pool in self.stage_pools:
await pool.abort_requests(request_ids)
stage_outputs = await pool.abort_requests(request_ids) or []
if bool(getattr(pool, "final_output", False)) and getattr(pool, "stage_type", None) != "diffusion":
final_output_type = getattr(pool.stage_client, "final_output_type", None) or "text"
for orch_req_id, request_output in stage_outputs:
req_state = self.request_states.get(orch_req_id)
if req_state is None:
continue
final_stage_ids = req_state.final_output_stage_ids or {req_state.final_stage_id}
if pool.stage_id not in final_stage_ids:
continue
engine_output = OmniRequestOutput.from_stage_output(
request_output,
request_id=orch_req_id,
finished=True,
stage_id=pool.stage_id,
replica_id=pool.get_bound_replica_id(orch_req_id),
final_output_type=final_output_type,
)
abort_outputs.append(
OutputMessage(
request_id=orch_req_id,
stage_id=pool.stage_id,
replica_id=pool.get_bound_replica_id(orch_req_id),
engine_outputs=engine_output,
metrics=None,
finished=False,
stage_submit_ts=req_state.stage_submit_ts.get(pool.stage_id),
)
)
pool.release_bindings(request_ids)
last_index_by_req: dict[str, int] = {}
for index, output_msg in enumerate(abort_outputs):
last_index_by_req[output_msg.request_id] = index
for index, output_msg in enumerate(abort_outputs):
output_msg.finished = index == last_index_by_req[output_msg.request_id]
return abort_outputs
def _release_request_bindings(self, request_ids: list[str]) -> None:
"""Release all stage-local route bindings for the given request ids."""
@@ -1688,7 +1737,7 @@ class Orchestrator:
*,
abort: bool = False,
close_duplex_sessions: bool = False,
) -> None:
) -> list[OutputMessage]:
"""Release pool bindings and logical request state for the given ids.
CFG-aware: cleaning a parent releases its tracker state and pulls its
@@ -1697,9 +1746,13 @@ class Orchestrator:
can never complete). Every teardown path (stage error, abort, replica
loss, membership unregister) funnels through here, so tracker state
cannot outlive its requests.
Returns:
Final-stage AR abort ``OutputMessage`` list when ``abort=True``;
otherwise an empty list.
"""
if not request_ids:
return
return []
cleanup_ids = list(dict.fromkeys(request_ids))
batch = set(cleanup_ids)
@@ -1741,9 +1794,10 @@ class Orchestrator:
cleanup_ids.extend(stale_request_ids)
cleanup_ids = list(dict.fromkeys(cleanup_ids))
abort_outputs: list[OutputMessage] = []
try:
if abort:
await self._abort_request_ids(cleanup_ids)
abort_outputs = await self._abort_request_ids(cleanup_ids)
self._release_request_bindings(cleanup_ids)
for request_id in cleanup_ids:
self._pd_kv_params.pop(request_id, None)
@@ -1759,6 +1813,7 @@ class Orchestrator:
raise
if closing_session_ids and self.duplex_control_plane is not None:
self.duplex_control_plane.finalize_closed_sessions(closing_session_ids)
return abort_outputs
async def _apply_raw_terminal_stage_finish(
self,
+54 -11
View File
@@ -1189,14 +1189,19 @@ class StagePool:
# ---- Stage-local control plane ----
async def abort_requests(self, request_ids: list[str]) -> None:
async def abort_requests(self, request_ids: list[str]) -> list[tuple[str, Any]]:
"""Abort the given requests in this stage pool.
Request-bound abort routing stays inside the pool because route affinity
(``request_id -> replica_id``) is pool-owned.
For AR stages, collect terminal abort ``RequestOutput`` objects from
output-processor state *before* EngineCore abort so partial tokens
generated so far are preserved. Diffusion stages return no abort
outputs (whole-sample retry remains the recovery path).
"""
if not request_ids:
return
return []
request_ids_by_replica: dict[int, list[str]] = {}
for request_id in request_ids:
@@ -1206,18 +1211,45 @@ class StagePool:
continue
request_ids_by_replica.setdefault(replica_id, []).append(request_id)
abort_outputs: list[tuple[str, Any]] = []
is_diffusion = self.stage_type == "diffusion"
for replica_id, replica_request_ids in request_ids_by_replica.items():
# Orchestrator ids are OP external ids; EngineCore may use a
# different internal request_id assigned by InputProcessor.
# Always key collected outputs by the orchestrator id we passed
# (internal=False), never by RequestOutput.request_id — some stages
# still surface EngineCore/internal ids and would drop the prefix.
engine_abort_ids = list(replica_request_ids)
if not is_diffusion and self._output_processor is not None:
collect = getattr(self._output_processor, "abort_requests_collecting_outputs", None)
if collect is not None:
engine_abort_ids = []
for orch_req_id in replica_request_ids:
collected_ids, stage_outputs = collect(
[orch_req_id],
internal=False,
commit_state=False,
)
for req_out in stage_outputs:
abort_outputs.append((orch_req_id, req_out))
if collected_ids:
engine_abort_ids.extend(collected_ids)
else:
engine_abort_ids.append(orch_req_id)
else:
aborted = self._output_processor.abort_requests(replica_request_ids, internal=False)
if aborted:
engine_abort_ids = list(aborted)
client = self.clients[replica_id]
if client is None:
continue
await client.abort_requests_async(replica_request_ids)
await client.abort_requests_async(engine_abort_ids)
if not is_diffusion and self._output_processor is not None:
commit = getattr(self._output_processor, "commit_aborted_request_state", None)
if callable(commit):
commit(replica_request_ids, internal=False)
# Clean up OutputProcessor state (e.g. mm_accumulated tensors) that
# would otherwise leak — aborted requests never produce a final
# EngineCoreOutput, so process_outputs() never fires its cleanup path.
all_aborted = [rid for ids in request_ids_by_replica.values() for rid in ids]
if all_aborted and self._output_processor is not None:
self._output_processor.abort_requests(all_aborted, internal=True)
return abort_outputs
async def collective_rpc(
self,
@@ -1227,7 +1259,12 @@ class StagePool:
args: tuple[Any, ...] = (),
kwargs: dict[str, Any] | None = None,
) -> dict[str, Any] | Any:
"""Dispatch a stage-scoped control-plane RPC to one physical route."""
"""Dispatch a stage-scoped control-plane RPC to one physical route.
EngineCore control methods (``pause_scheduler``, ``resume_scheduler``,
``sleep``, ``wake_up``) propagate worker exceptions. Capability probes
and other RPCs still return ``{"supported": False, "error": ...}``.
"""
args = tuple(args or ())
kwargs = dict(kwargs or {})
client = self.clients[replica_id]
@@ -1258,9 +1295,15 @@ class StagePool:
replica_id,
method,
)
if method in self._ENGINE_CORE_CONTROL_ASYNC_METHODS:
raise
if isinstance(exc, TimeoutError):
error = f"{type(exc).__name__}: {method} timed out after {timeout}s"
else:
error = str(exc) or repr(exc)
return {
"supported": False,
"error": str(exc),
"error": error,
}
def shutdown_replica(self, replica_id: int) -> None: