[Bugfix] Fix colocate-async sleep admission race and deliver AR abort tokens (#6367)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Signed-off-by: Cursor <cursoragent@cursor.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
23 files changed
+1229
-157
No files matched your search
@@ -67,6 +67,7 @@ from vllm_omni.engine.messages import (
|
||||
EngineQueueMessage,
|
||||
ErrorMessage,
|
||||
InteractionMessage,
|
||||
OutputMessage,
|
||||
StageSubmissionMessage,
|
||||
)
|
||||
from vllm_omni.engine.orchestrator import Orchestrator
|
||||
@@ -1920,12 +1921,16 @@ class AsyncOmniEngine:
|
||||
self,
|
||||
request_ids: list[str],
|
||||
timeout: float | None = None,
|
||||
) -> None:
|
||||
) -> list[OutputMessage]:
|
||||
"""Abort requests and wait for orchestrator acknowledgment.
|
||||
|
||||
Unlike :meth:`abort`, this generates an ``rpc_id``, correlates the
|
||||
:class:`AbortResultMessage` via :class:`CorrelatedRpcClient`, and
|
||||
raises if the orchestrator reports failure or times out.
|
||||
|
||||
Returns:
|
||||
Final-stage AR abort ``OutputMessage`` list carrying partial
|
||||
tokens generated before abort (empty for diffusion / no OP state).
|
||||
"""
|
||||
if not request_ids or getattr(self, "_shutdown_called", False):
|
||||
return
|
||||
@@ -1959,6 +1964,7 @@ class AsyncOmniEngine:
|
||||
raise
|
||||
if not result_msg.success:
|
||||
raise RuntimeError(result_msg.error or "abort failed")
|
||||
return list(result_msg.abort_outputs or [])
|
||||
|
||||
def submit_interaction(
|
||||
self,
|
||||
|
||||
@@ -48,17 +48,6 @@ class AbortRequestMessage(EngineQueueMessage, kw_only=True):
|
||||
rpc_id: str | None = None
|
||||
|
||||
|
||||
class AbortResultMessage(EngineQueueMessage, kw_only=True):
|
||||
type: Literal["abort_result"] = "abort_result"
|
||||
rpc_id: str
|
||||
success: bool
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def rpc_correlation_key(self) -> tuple[str, str]:
|
||||
return ("abort", self.rpc_id)
|
||||
|
||||
|
||||
class InteractionMessage(EngineQueueMessage, kw_only=True):
|
||||
type: Literal["interaction"] = "interaction"
|
||||
request_id: str
|
||||
@@ -113,6 +102,20 @@ class OutputMessage(EngineQueueMessage, kw_only=True):
|
||||
stage_submit_ts: float | None = None
|
||||
|
||||
|
||||
class AbortResultMessage(EngineQueueMessage, kw_only=True):
|
||||
type: Literal["abort_result"] = "abort_result"
|
||||
rpc_id: str
|
||||
success: bool
|
||||
error: str | None = None
|
||||
# Final-stage AR abort outputs (partial tokens) for frontend delivery.
|
||||
# Empty for diffusion / requests with no output-processor state.
|
||||
abort_outputs: list[OutputMessage] | None = None
|
||||
|
||||
@property
|
||||
def rpc_correlation_key(self) -> tuple[str, str]:
|
||||
return ("abort", self.rpc_id)
|
||||
|
||||
|
||||
class StageMetricsMessage(EngineQueueMessage, kw_only=True):
|
||||
type: Literal["stage_metrics"] = "stage_metrics"
|
||||
request_id: str
|
||||
|
||||
@@ -871,14 +871,19 @@ class Orchestrator:
|
||||
When ``msg.rpc_id`` is set, emit :class:`AbortResultMessage` after
|
||||
stage aborts, binding release, and request cleanup complete so the
|
||||
caller can await acknowledgment. Fire-and-forget aborts omit rpc_id.
|
||||
|
||||
AR final-stage abort outputs (partial tokens) are attached to the
|
||||
result message when ``rpc_id`` is set, or enqueued on
|
||||
``output_async_queue`` for fire-and-forget aborts.
|
||||
"""
|
||||
request_ids = msg.request_ids
|
||||
error: str | None = None
|
||||
abort_outputs: list[OutputMessage] = []
|
||||
try:
|
||||
# _cleanup_request_ids is CFG-aware: it expands aborted parents to
|
||||
# their companions and fails a deferred parent whose companion is
|
||||
# aborted before its output arrived.
|
||||
await self._cleanup_request_ids(list(request_ids), abort=True)
|
||||
abort_outputs = await self._cleanup_request_ids(list(request_ids), abort=True)
|
||||
logger.info("[Orchestrator] Aborted request(s) %s", request_ids)
|
||||
except Exception as exc:
|
||||
error = str(exc)
|
||||
@@ -893,8 +898,12 @@ class Orchestrator:
|
||||
rpc_id=msg.rpc_id,
|
||||
success=error is None,
|
||||
error=error,
|
||||
abort_outputs=abort_outputs or None,
|
||||
)
|
||||
)
|
||||
elif abort_outputs:
|
||||
for output_msg in abort_outputs:
|
||||
await self.output_async_queue.put(output_msg)
|
||||
|
||||
async def _handle_interaction(self, msg: InteractionMessage) -> None:
|
||||
"""Handle a midway interaction for an active streaming diffusion request."""
|
||||
@@ -934,13 +943,53 @@ class Orchestrator:
|
||||
)
|
||||
)
|
||||
|
||||
async def _abort_request_ids(self, request_ids: list[str]) -> None:
|
||||
"""Forward abort requests to all stage pools."""
|
||||
async def _abort_request_ids(self, request_ids: list[str]) -> list[OutputMessage]:
|
||||
"""Forward abort requests to all stage pools.
|
||||
|
||||
Collects final-stage AR abort outputs (partial tokens) while request
|
||||
state is still available for stage-id filtering. Diffusion pools do
|
||||
not contribute abort outputs.
|
||||
"""
|
||||
if not request_ids:
|
||||
return
|
||||
return []
|
||||
abort_outputs: list[OutputMessage] = []
|
||||
for pool in self.stage_pools:
|
||||
await pool.abort_requests(request_ids)
|
||||
stage_outputs = await pool.abort_requests(request_ids) or []
|
||||
if bool(getattr(pool, "final_output", False)) and getattr(pool, "stage_type", None) != "diffusion":
|
||||
final_output_type = getattr(pool.stage_client, "final_output_type", None) or "text"
|
||||
for orch_req_id, request_output in stage_outputs:
|
||||
req_state = self.request_states.get(orch_req_id)
|
||||
if req_state is None:
|
||||
continue
|
||||
final_stage_ids = req_state.final_output_stage_ids or {req_state.final_stage_id}
|
||||
if pool.stage_id not in final_stage_ids:
|
||||
continue
|
||||
engine_output = OmniRequestOutput.from_stage_output(
|
||||
request_output,
|
||||
request_id=orch_req_id,
|
||||
finished=True,
|
||||
stage_id=pool.stage_id,
|
||||
replica_id=pool.get_bound_replica_id(orch_req_id),
|
||||
final_output_type=final_output_type,
|
||||
)
|
||||
abort_outputs.append(
|
||||
OutputMessage(
|
||||
request_id=orch_req_id,
|
||||
stage_id=pool.stage_id,
|
||||
replica_id=pool.get_bound_replica_id(orch_req_id),
|
||||
engine_outputs=engine_output,
|
||||
metrics=None,
|
||||
finished=False,
|
||||
stage_submit_ts=req_state.stage_submit_ts.get(pool.stage_id),
|
||||
)
|
||||
)
|
||||
pool.release_bindings(request_ids)
|
||||
last_index_by_req: dict[str, int] = {}
|
||||
for index, output_msg in enumerate(abort_outputs):
|
||||
last_index_by_req[output_msg.request_id] = index
|
||||
for index, output_msg in enumerate(abort_outputs):
|
||||
output_msg.finished = index == last_index_by_req[output_msg.request_id]
|
||||
return abort_outputs
|
||||
|
||||
def _release_request_bindings(self, request_ids: list[str]) -> None:
|
||||
"""Release all stage-local route bindings for the given request ids."""
|
||||
@@ -1688,7 +1737,7 @@ class Orchestrator:
|
||||
*,
|
||||
abort: bool = False,
|
||||
close_duplex_sessions: bool = False,
|
||||
) -> None:
|
||||
) -> list[OutputMessage]:
|
||||
"""Release pool bindings and logical request state for the given ids.
|
||||
|
||||
CFG-aware: cleaning a parent releases its tracker state and pulls its
|
||||
@@ -1697,9 +1746,13 @@ class Orchestrator:
|
||||
can never complete). Every teardown path (stage error, abort, replica
|
||||
loss, membership unregister) funnels through here, so tracker state
|
||||
cannot outlive its requests.
|
||||
|
||||
Returns:
|
||||
Final-stage AR abort ``OutputMessage`` list when ``abort=True``;
|
||||
otherwise an empty list.
|
||||
"""
|
||||
if not request_ids:
|
||||
return
|
||||
return []
|
||||
|
||||
cleanup_ids = list(dict.fromkeys(request_ids))
|
||||
batch = set(cleanup_ids)
|
||||
@@ -1741,9 +1794,10 @@ class Orchestrator:
|
||||
cleanup_ids.extend(stale_request_ids)
|
||||
cleanup_ids = list(dict.fromkeys(cleanup_ids))
|
||||
|
||||
abort_outputs: list[OutputMessage] = []
|
||||
try:
|
||||
if abort:
|
||||
await self._abort_request_ids(cleanup_ids)
|
||||
abort_outputs = await self._abort_request_ids(cleanup_ids)
|
||||
self._release_request_bindings(cleanup_ids)
|
||||
for request_id in cleanup_ids:
|
||||
self._pd_kv_params.pop(request_id, None)
|
||||
@@ -1759,6 +1813,7 @@ class Orchestrator:
|
||||
raise
|
||||
if closing_session_ids and self.duplex_control_plane is not None:
|
||||
self.duplex_control_plane.finalize_closed_sessions(closing_session_ids)
|
||||
return abort_outputs
|
||||
|
||||
async def _apply_raw_terminal_stage_finish(
|
||||
self,
|
||||
|
||||
@@ -1189,14 +1189,19 @@ class StagePool:
|
||||
|
||||
# ---- Stage-local control plane ----
|
||||
|
||||
async def abort_requests(self, request_ids: list[str]) -> None:
|
||||
async def abort_requests(self, request_ids: list[str]) -> list[tuple[str, Any]]:
|
||||
"""Abort the given requests in this stage pool.
|
||||
|
||||
Request-bound abort routing stays inside the pool because route affinity
|
||||
(``request_id -> replica_id``) is pool-owned.
|
||||
|
||||
For AR stages, collect terminal abort ``RequestOutput`` objects from
|
||||
output-processor state *before* EngineCore abort so partial tokens
|
||||
generated so far are preserved. Diffusion stages return no abort
|
||||
outputs (whole-sample retry remains the recovery path).
|
||||
"""
|
||||
if not request_ids:
|
||||
return
|
||||
return []
|
||||
|
||||
request_ids_by_replica: dict[int, list[str]] = {}
|
||||
for request_id in request_ids:
|
||||
@@ -1206,18 +1211,45 @@ class StagePool:
|
||||
continue
|
||||
request_ids_by_replica.setdefault(replica_id, []).append(request_id)
|
||||
|
||||
abort_outputs: list[tuple[str, Any]] = []
|
||||
is_diffusion = self.stage_type == "diffusion"
|
||||
for replica_id, replica_request_ids in request_ids_by_replica.items():
|
||||
# Orchestrator ids are OP external ids; EngineCore may use a
|
||||
# different internal request_id assigned by InputProcessor.
|
||||
# Always key collected outputs by the orchestrator id we passed
|
||||
# (internal=False), never by RequestOutput.request_id — some stages
|
||||
# still surface EngineCore/internal ids and would drop the prefix.
|
||||
engine_abort_ids = list(replica_request_ids)
|
||||
if not is_diffusion and self._output_processor is not None:
|
||||
collect = getattr(self._output_processor, "abort_requests_collecting_outputs", None)
|
||||
if collect is not None:
|
||||
engine_abort_ids = []
|
||||
for orch_req_id in replica_request_ids:
|
||||
collected_ids, stage_outputs = collect(
|
||||
[orch_req_id],
|
||||
internal=False,
|
||||
commit_state=False,
|
||||
)
|
||||
for req_out in stage_outputs:
|
||||
abort_outputs.append((orch_req_id, req_out))
|
||||
if collected_ids:
|
||||
engine_abort_ids.extend(collected_ids)
|
||||
else:
|
||||
engine_abort_ids.append(orch_req_id)
|
||||
else:
|
||||
aborted = self._output_processor.abort_requests(replica_request_ids, internal=False)
|
||||
if aborted:
|
||||
engine_abort_ids = list(aborted)
|
||||
client = self.clients[replica_id]
|
||||
if client is None:
|
||||
continue
|
||||
await client.abort_requests_async(replica_request_ids)
|
||||
await client.abort_requests_async(engine_abort_ids)
|
||||
if not is_diffusion and self._output_processor is not None:
|
||||
commit = getattr(self._output_processor, "commit_aborted_request_state", None)
|
||||
if callable(commit):
|
||||
commit(replica_request_ids, internal=False)
|
||||
|
||||
# Clean up OutputProcessor state (e.g. mm_accumulated tensors) that
|
||||
# would otherwise leak — aborted requests never produce a final
|
||||
# EngineCoreOutput, so process_outputs() never fires its cleanup path.
|
||||
all_aborted = [rid for ids in request_ids_by_replica.values() for rid in ids]
|
||||
if all_aborted and self._output_processor is not None:
|
||||
self._output_processor.abort_requests(all_aborted, internal=True)
|
||||
return abort_outputs
|
||||
|
||||
async def collective_rpc(
|
||||
self,
|
||||
@@ -1227,7 +1259,12 @@ class StagePool:
|
||||
args: tuple[Any, ...] = (),
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | Any:
|
||||
"""Dispatch a stage-scoped control-plane RPC to one physical route."""
|
||||
"""Dispatch a stage-scoped control-plane RPC to one physical route.
|
||||
|
||||
EngineCore control methods (``pause_scheduler``, ``resume_scheduler``,
|
||||
``sleep``, ``wake_up``) propagate worker exceptions. Capability probes
|
||||
and other RPCs still return ``{"supported": False, "error": ...}``.
|
||||
"""
|
||||
args = tuple(args or ())
|
||||
kwargs = dict(kwargs or {})
|
||||
client = self.clients[replica_id]
|
||||
@@ -1258,9 +1295,15 @@ class StagePool:
|
||||
replica_id,
|
||||
method,
|
||||
)
|
||||
if method in self._ENGINE_CORE_CONTROL_ASYNC_METHODS:
|
||||
raise
|
||||
if isinstance(exc, TimeoutError):
|
||||
error = f"{type(exc).__name__}: {method} timed out after {timeout}s"
|
||||
else:
|
||||
error = str(exc) or repr(exc)
|
||||
return {
|
||||
"supported": False,
|
||||
"error": str(exc),
|
||||
"error": error,
|
||||
}
|
||||
|
||||
def shutdown_replica(self, replica_id: int) -> None:
|
||||
|
||||
Reference in new issue
Block a user