[Bugfix] Fix colocate-async sleep admission race and deliver AR abort tokens (#6367)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Signed-off-by: Cursor <cursoragent@cursor.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Samit <285365963@qq.com>
This commit is contained in:
23 files changed
+1229
-157
No files matched your search
+272
-102
@@ -17,7 +17,7 @@ from vllm import TokensPrompt
|
||||
from vllm.engine.protocol import EngineClient, StreamingInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.outputs import CompletionOutput, PoolingRequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
@@ -150,12 +150,19 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
OmniBase.__init__(self, model=model, **kwargs)
|
||||
self._pause_cond: asyncio.Condition = asyncio.Condition()
|
||||
self._paused: bool = False
|
||||
# In-flight EngineCore submits (non-streaming add_request, or each
|
||||
# streaming ADD/update/final marker). sleep() waits for this to hit
|
||||
# zero so a pipelined request cannot race into EngineCore during
|
||||
# drain/offload. Streaming generate does not hold a slot while
|
||||
# waiting for the next client chunk.
|
||||
self._admitting: int = 0
|
||||
# True after pause_generation() or AR EngineCore sleep; wake_up must
|
||||
# not reopen generate() until resume_generation(). Diffusion-only
|
||||
# sleep uses _paused as a temporary admission gate and clears it
|
||||
# on wake so sleep → wake → generate keeps working.
|
||||
self._hold_admission_until_resume: bool = False
|
||||
self._sleeping_tags: set[str] = set()
|
||||
self._stage_sleeping_tags: dict[int, set[str]] = {}
|
||||
self._level2_sleeping: bool = False
|
||||
self._duplex_request_client: DuplexRequestClient | None = None
|
||||
self.duplex_lifecycle_events: asyncio.Queue[DuplexSessionLifecycleMessage] = asyncio.Queue()
|
||||
@@ -513,31 +520,40 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
external_request_id = request_id
|
||||
request_id = self._get_unique_request_id(external_request_id)
|
||||
|
||||
# Wait until generation is resumed if the engine is paused
|
||||
# Wait until generation is resumed if the engine is paused. Non-streaming
|
||||
# generate holds an admission slot until add_request completes so sleep()
|
||||
# cannot race a just-unblocked generate into EngineCore during offload.
|
||||
# Streaming generate does **not** hold the slot while waiting for the
|
||||
# first client chunk; the input pump acquires it immediately before each
|
||||
# EngineCore ADD/update.
|
||||
streaming_input = isinstance(prompt, AsyncGenerator)
|
||||
async with self._pause_cond:
|
||||
await self._pause_cond.wait_for(lambda: not self._paused)
|
||||
if not streaming_input:
|
||||
self._admitting = getattr(self, "_admitting", 0) + 1
|
||||
admitting = not streaming_input
|
||||
|
||||
logger.debug(f"[AsyncOmni] generate() called for request {external_request_id}")
|
||||
|
||||
_sleeping_tags = getattr(self, "_sleeping_tags", None)
|
||||
if _sleeping_tags:
|
||||
raise RuntimeError(
|
||||
f"Generation rejected: Engine is partially or fully asleep. "
|
||||
f"Currently sleeping tags: {list(_sleeping_tags)}. "
|
||||
f"Please perform a full wake_up before generating."
|
||||
)
|
||||
|
||||
# Reject diffusion list-prompt early with a clear API error.
|
||||
if isinstance(prompt, list) and any(
|
||||
getattr(client, "stage_type", "") == "diffusion" for client in getattr(self.engine, "stage_clients", [])
|
||||
):
|
||||
raise ValueError(
|
||||
"Diffusion stages accept only a single prompt per request. "
|
||||
"Submit multiple independent requests to use scheduler batching."
|
||||
)
|
||||
|
||||
input_stream_task: asyncio.Task | None = None
|
||||
try:
|
||||
_sleeping_tags = getattr(self, "_sleeping_tags", None)
|
||||
if _sleeping_tags:
|
||||
raise RuntimeError(
|
||||
f"Generation rejected: Engine is partially or fully asleep. "
|
||||
f"Currently sleeping tags: {list(_sleeping_tags)}. "
|
||||
f"Please perform a full wake_up before generating."
|
||||
)
|
||||
|
||||
# Reject diffusion list-prompt early with a clear API error.
|
||||
if isinstance(prompt, list) and any(
|
||||
getattr(client, "stage_type", "") == "diffusion" for client in getattr(self.engine, "stage_clients", [])
|
||||
):
|
||||
raise ValueError(
|
||||
"Diffusion stages accept only a single prompt per request. "
|
||||
"Submit multiple independent requests to use scheduler batching."
|
||||
)
|
||||
|
||||
# Start final output dispatcher on the first call to generate()
|
||||
self._final_output_handler()
|
||||
|
||||
@@ -598,8 +614,11 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
req_sp_list[p_id] = self._prepare_prefill_sampling_params(request_id, req_sp_list[p_id])
|
||||
|
||||
# Add request(s) to stage 0. For streaming inputs, submit
|
||||
# chunks incrementally through streaming_update.
|
||||
if isinstance(prompt, AsyncGenerator):
|
||||
# chunks incrementally through streaming_update. The helper
|
||||
# returns as soon as the pump task is created; each ADD/update
|
||||
# takes its own admission slot inside the pump.
|
||||
if streaming_input:
|
||||
first_chunk_submitted = asyncio.get_running_loop().create_future()
|
||||
input_stream_task = await self._add_streaming_input_request(
|
||||
request_id=request_id,
|
||||
input_stream=prompt,
|
||||
@@ -608,7 +627,9 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=wall_start_ts,
|
||||
lora_request=lora_request,
|
||||
first_chunk_submitted=first_chunk_submitted,
|
||||
)
|
||||
await first_chunk_submitted
|
||||
else:
|
||||
await self.engine.add_request_async(
|
||||
request_id=request_id,
|
||||
@@ -622,6 +643,9 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
submit_ts = time.time()
|
||||
req_state.metrics.stage_first_ts[0] = submit_ts
|
||||
req_start_ts[request_id] = submit_ts
|
||||
if admitting:
|
||||
await self._release_generate_admission()
|
||||
admitting = False
|
||||
# Refresh gauges on arrival.
|
||||
self._publish_request_gauges(len(self.request_states))
|
||||
|
||||
@@ -640,18 +664,7 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
|
||||
logger.debug(f"[AsyncOmni] Request {request_id} completed")
|
||||
|
||||
# The input pump can outlive a normally-completed generation
|
||||
# (e.g. a realtime client that keeps streaming after the final
|
||||
# output); a live pump keeps issuing streaming updates for a
|
||||
# request the orchestrator no longer tracks.
|
||||
if input_stream_task is not None and not input_stream_task.done():
|
||||
input_stream_task.cancel()
|
||||
|
||||
self._log_summary_and_cleanup(request_id)
|
||||
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
if input_stream_task is not None and not input_stream_task.done():
|
||||
input_stream_task.cancel()
|
||||
self._record_request_failure_once(request_id, reason="client_disconnect")
|
||||
await self._abort_internal_requests(request_id)
|
||||
logger.info(f"[AsyncOmni] Request {request_id} aborted.")
|
||||
@@ -661,6 +674,28 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
await self._abort_internal_requests(request_id)
|
||||
logger.info(f"[AsyncOmni] Request {request_id} failed (input error): {e}")
|
||||
raise
|
||||
finally:
|
||||
if input_stream_task is not None and not input_stream_task.done():
|
||||
input_stream_task.cancel()
|
||||
if admitting:
|
||||
await self._release_generate_admission()
|
||||
self._log_summary_and_cleanup(request_id)
|
||||
|
||||
async def _release_generate_admission(self) -> None:
|
||||
"""Drop one in-flight generate admission slot held across add_request."""
|
||||
async with self._pause_cond:
|
||||
self._admitting = max(getattr(self, "_admitting", 1) - 1, 0)
|
||||
self._pause_cond.notify_all()
|
||||
|
||||
async def _submit_with_admission(self, awaitable):
|
||||
"""Wait for resume, hold one admission slot for a single EngineCore submit."""
|
||||
async with self._pause_cond:
|
||||
await self._pause_cond.wait_for(lambda: not self._paused)
|
||||
self._admitting = getattr(self, "_admitting", 0) + 1
|
||||
try:
|
||||
return await awaitable
|
||||
finally:
|
||||
await self._release_generate_admission()
|
||||
|
||||
async def _add_streaming_input_request(
|
||||
self,
|
||||
@@ -672,6 +707,7 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
final_output_stage_ids: Sequence[int],
|
||||
arrival_time: float,
|
||||
lora_request: Any = None,
|
||||
first_chunk_submitted: asyncio.Future[None] | None = None,
|
||||
) -> asyncio.Task:
|
||||
"""Submit a streaming input generator as incremental stage-0 updates."""
|
||||
if not sampling_params_list:
|
||||
@@ -688,6 +724,10 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
stage0_params = stage0_params.clone()
|
||||
stage0_params.skip_clone = True
|
||||
|
||||
def _mark_first_chunk_submitted() -> None:
|
||||
if first_chunk_submitted is not None and not first_chunk_submitted.done():
|
||||
first_chunk_submitted.set_result(None)
|
||||
|
||||
async def handle_inputs() -> None:
|
||||
nonlocal has_submitted_first_chunk
|
||||
cancelled = False
|
||||
@@ -701,29 +741,34 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
prompt_text, _, _ = extract_prompt_components(self.model_config, chunk_prompt)
|
||||
|
||||
if not has_submitted_first_chunk:
|
||||
await self.engine.add_request_async(
|
||||
request_id=request_id,
|
||||
prompt=chunk_prompt,
|
||||
prompt_text=prompt_text,
|
||||
sampling_params_list=chunk_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=True,
|
||||
await self._submit_with_admission(
|
||||
self.engine.add_request_async(
|
||||
request_id=request_id,
|
||||
prompt=chunk_prompt,
|
||||
prompt_text=prompt_text,
|
||||
sampling_params_list=chunk_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=True,
|
||||
)
|
||||
)
|
||||
has_submitted_first_chunk = True
|
||||
_mark_first_chunk_submitted()
|
||||
else:
|
||||
await self.engine.add_streaming_update_async(
|
||||
request_id=request_id,
|
||||
prompt=chunk_prompt,
|
||||
prompt_text=prompt_text,
|
||||
sampling_params_list=chunk_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=True,
|
||||
await self._submit_with_admission(
|
||||
self.engine.add_streaming_update_async(
|
||||
request_id=request_id,
|
||||
prompt=chunk_prompt,
|
||||
prompt_text=prompt_text,
|
||||
sampling_params_list=chunk_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=True,
|
||||
)
|
||||
)
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
cancelled = True
|
||||
@@ -738,37 +783,47 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if not cancelled:
|
||||
# Send empty final request to indicate that inputs have
|
||||
# finished. Don't send if canceled (session was aborted).
|
||||
final_sampling_params_list = list(sampling_params_list)
|
||||
final_sampling_params_list[0] = stage0_params
|
||||
final_prompt = TokensPrompt(prompt_token_ids=[0])
|
||||
try:
|
||||
if not cancelled:
|
||||
# Send empty final request to indicate that inputs have
|
||||
# finished. Don't send if canceled (session was aborted).
|
||||
final_sampling_params_list = list(sampling_params_list)
|
||||
final_sampling_params_list[0] = stage0_params
|
||||
final_prompt = TokensPrompt(prompt_token_ids=[0])
|
||||
|
||||
if has_submitted_first_chunk:
|
||||
await self.engine.add_streaming_update_async(
|
||||
request_id=request_id,
|
||||
prompt=final_prompt,
|
||||
prompt_text=None,
|
||||
sampling_params_list=final_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=False,
|
||||
)
|
||||
else:
|
||||
await self.engine.add_request_async(
|
||||
request_id=request_id,
|
||||
prompt=final_prompt,
|
||||
prompt_text=None,
|
||||
sampling_params_list=final_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=False,
|
||||
)
|
||||
if has_submitted_first_chunk:
|
||||
await self._submit_with_admission(
|
||||
self.engine.add_streaming_update_async(
|
||||
request_id=request_id,
|
||||
prompt=final_prompt,
|
||||
prompt_text=None,
|
||||
sampling_params_list=final_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=False,
|
||||
)
|
||||
)
|
||||
else:
|
||||
await self._submit_with_admission(
|
||||
self.engine.add_request_async(
|
||||
request_id=request_id,
|
||||
prompt=final_prompt,
|
||||
prompt_text=None,
|
||||
sampling_params_list=final_sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
arrival_time=arrival_time,
|
||||
lora_request=lora_request,
|
||||
resumable=False,
|
||||
)
|
||||
)
|
||||
has_submitted_first_chunk = True
|
||||
finally:
|
||||
# Unblock generate() even on cancel / empty stream / submit
|
||||
# failure so it can observe a terminal abort or empty result.
|
||||
_mark_first_chunk_submitted()
|
||||
|
||||
input_stream_task = asyncio.create_task(handle_inputs())
|
||||
req_state.input_stream_task = input_stream_task
|
||||
@@ -1129,22 +1184,76 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
await self._abort(internal_req_ids)
|
||||
|
||||
async def _abort(self, request_ids: list[str]) -> None:
|
||||
"""Abort request IDs via the engine and clean frontend state after ack.
|
||||
"""Abort request IDs via the engine and enqueue terminal abort outputs.
|
||||
|
||||
Waits for orchestrator abort acknowledgment before popping
|
||||
``request_states`` so generate() cleanup stays consistent with
|
||||
backend binding/request teardown. Orchestrator abort errors propagate.
|
||||
Waits for orchestrator abort acknowledgment, enqueues any AR terminal
|
||||
abort outputs (partial tokens) into each request's asyncio queue, then
|
||||
cancels the input pump. Frontend ``request_states`` stay registered so
|
||||
``generate()`` can consume the terminal message in
|
||||
``_process_orchestrator_results`` and run normal cleanup.
|
||||
|
||||
When ``abort_async`` returns no output for an active request (OP not
|
||||
registered yet, unbound replica, or orchestrator id drop), enqueue a
|
||||
synthetic finished abort so ``generate()`` cannot hang on ``queue.get``.
|
||||
"""
|
||||
await self.engine.abort_async(request_ids)
|
||||
abort_outputs = await self.engine.abort_async(request_ids) or []
|
||||
delivered: set[str] = set()
|
||||
for output_msg in abort_outputs:
|
||||
req_id = getattr(output_msg, "request_id", None)
|
||||
if req_id is None:
|
||||
continue
|
||||
state = self.request_states.get(req_id)
|
||||
if state is None:
|
||||
logger.debug("[AsyncOmni] Dropping abort output for unknown req %s", req_id)
|
||||
continue
|
||||
await state.queue.put(output_msg)
|
||||
delivered.add(req_id)
|
||||
for rid in request_ids:
|
||||
state = self.request_states.get(rid)
|
||||
if state is not None and rid not in delivered:
|
||||
queue = getattr(state, "queue", None)
|
||||
if queue is not None:
|
||||
await state.queue.put(self._synthetic_abort_output_message(rid))
|
||||
delivered.add(rid)
|
||||
for rid in request_ids:
|
||||
self._record_request_failure_once(rid, reason="client_abort")
|
||||
state = self.request_states.pop(rid, None)
|
||||
state = self.request_states.get(rid)
|
||||
input_stream_task = getattr(state, "input_stream_task", None)
|
||||
if input_stream_task is not None and not input_stream_task.done():
|
||||
input_stream_task.cancel()
|
||||
if self.log_stats:
|
||||
logger.info("[AsyncOmni] Aborted request(s) %s", ",".join(request_ids))
|
||||
|
||||
@staticmethod
|
||||
def _synthetic_abort_output_message(request_id: str) -> OutputMessage:
|
||||
"""Terminal abort OutputMessage used when the engine returned none."""
|
||||
engine_output = OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
finished=True,
|
||||
stage_id=0,
|
||||
final_output_type="text",
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="abort",
|
||||
stop_reason=None,
|
||||
)
|
||||
],
|
||||
)
|
||||
return OutputMessage(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
replica_id=None,
|
||||
engine_outputs=engine_output,
|
||||
metrics=None,
|
||||
finished=True,
|
||||
stage_submit_ts=None,
|
||||
)
|
||||
|
||||
def _split_stage_ids_by_type(self, stage_ids: list[int] | None = None) -> tuple[list[int], list[int]]:
|
||||
"""Split stage ids into AR/LLM (EngineCore) vs diffusion (worker RPC)."""
|
||||
n_stages = len(self.engine.stage_clients)
|
||||
@@ -1168,6 +1277,47 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
ar_stage_ids.append(sid)
|
||||
return ar_stage_ids, diffusion_stage_ids
|
||||
|
||||
def _sleeping_tags_for_stages(self, stage_ids: list[int]) -> set[str]:
|
||||
"""Union of sleeping tags recorded for the given stages."""
|
||||
per_stage = getattr(self, "_stage_sleeping_tags", None) or {}
|
||||
tags: set[str] = set()
|
||||
for sid in stage_ids:
|
||||
tags.update(per_stage.get(sid, set()))
|
||||
return tags
|
||||
|
||||
def _refresh_union_sleeping_tags(self) -> None:
|
||||
"""Keep ``_sleeping_tags`` as the engine-wide union of per-stage tags."""
|
||||
per_stage = getattr(self, "_stage_sleeping_tags", None) or {}
|
||||
union: set[str] = set()
|
||||
for stage_tags in per_stage.values():
|
||||
union.update(stage_tags)
|
||||
self._sleeping_tags = union
|
||||
|
||||
def _record_stage_sleep(self, stage_ids: list[int], tags: Iterable[str]) -> None:
|
||||
per_stage = getattr(self, "_stage_sleeping_tags", None)
|
||||
if per_stage is None:
|
||||
per_stage = {}
|
||||
self._stage_sleeping_tags = per_stage
|
||||
tag_set = set(tags)
|
||||
for sid in stage_ids:
|
||||
per_stage.setdefault(sid, set()).update(tag_set)
|
||||
self._refresh_union_sleeping_tags()
|
||||
|
||||
def _clear_stage_sleep(self, stage_ids: list[int], tags: Iterable[str]) -> None:
|
||||
per_stage = getattr(self, "_stage_sleeping_tags", None)
|
||||
if per_stage is None:
|
||||
self._sleeping_tags = set()
|
||||
return
|
||||
tag_set = set(tags)
|
||||
for sid in stage_ids:
|
||||
remaining = per_stage.get(sid)
|
||||
if remaining is None:
|
||||
continue
|
||||
remaining.difference_update(tag_set)
|
||||
if not remaining:
|
||||
per_stage.pop(sid, None)
|
||||
self._refresh_union_sleeping_tags()
|
||||
|
||||
async def pause_generation(
|
||||
self,
|
||||
*,
|
||||
@@ -1266,11 +1416,24 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
return await self.collective_rpc(method="profile", args=(False, None), stage_ids=stages)
|
||||
|
||||
async def reset_mm_cache(self) -> None:
|
||||
"""Reset the multi-modal cache for all stages.
|
||||
"""Reset the frontend (P0) multimodal processor cache.
|
||||
|
||||
TODO: Forward to Orchestrator process via message.
|
||||
``EngineCore.sleep(level>=1)`` already clears the P1 receiver cache.
|
||||
Clearing P0 avoids hash-only follow-up requests after that reset.
|
||||
"""
|
||||
logger.warning("[AsyncOmni] reset_mm_cache not yet supported with Orchestrator process")
|
||||
processor = getattr(self, "input_processor", None)
|
||||
if processor is None:
|
||||
processor = getattr(self.engine, "input_processor", None)
|
||||
cache = getattr(processor, "mm_processor_cache", None)
|
||||
if cache is None:
|
||||
logger.debug("[AsyncOmni] reset_mm_cache: no frontend mm_processor_cache")
|
||||
return
|
||||
for name in ("clear", "reset", "clear_cache"):
|
||||
fn = getattr(cache, name, None)
|
||||
if callable(fn):
|
||||
fn()
|
||||
return
|
||||
logger.debug("[AsyncOmni] reset_mm_cache: cache has no clear/reset method")
|
||||
|
||||
async def reset_encoder_cache(self) -> None:
|
||||
"""Reset the encoder cache for all stages.
|
||||
@@ -1314,10 +1477,15 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
admission and ``sleep → wake → generate`` keeps working.
|
||||
"""
|
||||
# Block admission before any sleep RPC so generate() waits on
|
||||
# _pause_cond during the drain/offload window. EngineCore.sleep will
|
||||
# pause the AR scheduler itself; avoid a second pause_scheduler here.
|
||||
# _pause_cond during the drain/offload window. Wait until generate()
|
||||
# coroutines that already passed the pause check have submitted (or
|
||||
# failed) so EngineCore does not see ADD frames while sleeping.
|
||||
async with self._pause_cond:
|
||||
self._paused = True
|
||||
await self._pause_cond.wait_for(lambda: getattr(self, "_admitting", 0) == 0)
|
||||
|
||||
# P0 sender cache must drop hashes before EngineCore.sleep clears P1.
|
||||
await self.reset_mm_cache()
|
||||
|
||||
self._final_output_handler()
|
||||
ar_stage_ids, diffusion_stage_ids = self._split_stage_ids_by_type(stage_ids)
|
||||
@@ -1353,9 +1521,10 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
if diffusion_stage_ids:
|
||||
final_acks.extend(await self._sleep_diffusion(diffusion_stage_ids, level))
|
||||
|
||||
if not hasattr(self, "_sleeping_tags"):
|
||||
self._sleeping_tags = set()
|
||||
self._sleeping_tags.update([CuMemTag.WEIGHTS.value, CuMemTag.KV_CACHE.value])
|
||||
self._record_stage_sleep(
|
||||
ar_stage_ids + diffusion_stage_ids,
|
||||
[CuMemTag.WEIGHTS.value, CuMemTag.KV_CACHE.value],
|
||||
)
|
||||
if level == 2:
|
||||
self._level2_sleeping = True
|
||||
return final_acks
|
||||
@@ -1399,7 +1568,12 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
"Use sleep(level=1) instead, which offloads weights to CPU RAM "
|
||||
"and supports fast DMA restore."
|
||||
)
|
||||
_current_tags = getattr(self, "_sleeping_tags", set())
|
||||
ar_stage_ids, diffusion_stage_ids = self._split_stage_ids_by_type(stage_ids)
|
||||
target_stage_ids = ar_stage_ids + diffusion_stage_ids
|
||||
_current_tags = self._sleeping_tags_for_stages(target_stage_ids)
|
||||
per_stage = getattr(self, "_stage_sleeping_tags", None) or {}
|
||||
if not _current_tags and not per_stage:
|
||||
_current_tags = set(getattr(self, "_sleeping_tags", set()))
|
||||
if tags is None:
|
||||
requested_tags = list(_current_tags)
|
||||
else:
|
||||
@@ -1408,8 +1582,6 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
logger.info(f"[{self._name}] Requested tags {tags} are already warm. Skipping wake_up.")
|
||||
return []
|
||||
|
||||
ar_stage_ids, diffusion_stage_ids = self._split_stage_ids_by_type(stage_ids)
|
||||
|
||||
final_acks: list[OmniACK] = []
|
||||
if ar_stage_ids:
|
||||
logger.info("[%s] Waking AR stage(s) %s via EngineCore", self._name, ar_stage_ids)
|
||||
@@ -1433,9 +1605,7 @@ class AsyncOmni(EngineClient, OmniBase):
|
||||
if diffusion_stage_ids:
|
||||
final_acks.extend(await self._wake_diffusion(diffusion_stage_ids, requested_tags))
|
||||
|
||||
for t in requested_tags:
|
||||
if hasattr(self, "_sleeping_tags"):
|
||||
self._sleeping_tags.discard(t)
|
||||
self._clear_stage_sleep(target_stage_ids, requested_tags)
|
||||
# Only clear the level-2 flag once all tags are warm, in case partial
|
||||
# wake support (e.g. tags=["kv_cache"] only) is added in the future.
|
||||
if not getattr(self, "_sleeping_tags", None):
|
||||
|
||||
@@ -10,19 +10,26 @@ from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default for ``visible_baseline``: read the live device-control env var.
|
||||
# Explicit ``None`` means the caller captured an unset env and must not fall
|
||||
# back to a later thread's temporary CUDA_VISIBLE_DEVICES.
|
||||
_LIVE_DEVICE_ENV = object()
|
||||
|
||||
|
||||
def resolve_stage_physical_devices(
|
||||
stage_id: int,
|
||||
devices: str | int | None,
|
||||
*,
|
||||
visible_baseline: str | None = None,
|
||||
visible_baseline: str | None | object = _LIVE_DEVICE_ENV,
|
||||
) -> str | None:
|
||||
"""Map logical stage devices to physical IDs without mutating process env.
|
||||
|
||||
When ``visible_baseline`` is provided it is used for logical-to-physical
|
||||
mapping instead of the current device-control env var. This keeps parallel
|
||||
stage initialization correct even if another thread has temporarily
|
||||
narrowed ``CUDA_VISIBLE_DEVICES`` for a different stage.
|
||||
``visible_baseline`` is the device-control env captured at the start of
|
||||
replica init. Pass the captured string, or ``None`` when that env was
|
||||
unset. Omitting the argument reads the live env (single-threaded callers).
|
||||
|
||||
Parallel stage init must pass the captured value: another thread may have
|
||||
already narrowed ``CUDA_VISIBLE_DEVICES`` for a different stage.
|
||||
"""
|
||||
if devices in (None, "cpu"):
|
||||
return None
|
||||
@@ -32,7 +39,7 @@ def resolve_stage_physical_devices(
|
||||
|
||||
device_list = _parse_device_list(devices)
|
||||
env_var = current_omni_platform.device_control_env_var
|
||||
visible_devices = visible_baseline if visible_baseline is not None else os.environ.get(env_var)
|
||||
visible_devices = os.environ.get(env_var) if visible_baseline is _LIVE_DEVICE_ENV else visible_baseline
|
||||
if visible_devices is not None:
|
||||
visible_device_list = _parse_device_list(visible_devices)
|
||||
device_list = _map_device_list(stage_id, device_list, visible_device_list)
|
||||
@@ -43,7 +50,7 @@ def set_stage_devices(
|
||||
stage_id: int,
|
||||
devices: str | int | None,
|
||||
*,
|
||||
visible_baseline: str | None = None,
|
||||
visible_baseline: str | None | object = _LIVE_DEVICE_ENV,
|
||||
) -> str | None:
|
||||
"""Configure per-stage device visibility and current device (CUDA or NPU).
|
||||
|
||||
|
||||
Reference in new issue
Block a user