Merge main into dev/vllm-align for vLLM 0.31 release
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
commit
008fa25263
252 files changed
+27589
-1237
No files matched your search
@@ -198,6 +198,7 @@ class OmniEngineArgs(EngineArgs):
|
||||
async_chunk: bool = False
|
||||
session_mode: str = "turn"
|
||||
retains_state_across_chunks: bool = False
|
||||
supports_running_prefix_cache_reset: bool = True
|
||||
use_v2_model_runner: bool = False
|
||||
supports_native_mrv2_data_plane: bool = False
|
||||
# WS-A: Stage-1 active stream slots. 0 = legacy preempt-everything.
|
||||
@@ -432,6 +433,7 @@ class OmniEngineArgs(EngineArgs):
|
||||
async_chunk=self.async_chunk,
|
||||
session_mode=self.session_mode,
|
||||
retains_state_across_chunks=self.retains_state_across_chunks,
|
||||
supports_running_prefix_cache_reset=self.supports_running_prefix_cache_reset,
|
||||
use_v2_model_runner=self.use_v2_model_runner,
|
||||
supports_native_mrv2_data_plane=self.supports_native_mrv2_data_plane,
|
||||
active_stream_window=self.active_stream_window,
|
||||
@@ -560,6 +562,7 @@ class OrchestratorArgs:
|
||||
# === Diffusion model config ===
|
||||
num_gpus: int | None = None
|
||||
model_class_name: str | None = None
|
||||
hsdp_weight_load_strategy: str | None = None
|
||||
diffusion_load_format: str | None = None
|
||||
lora_path: list[str] | None = None
|
||||
lora_backend: str | None = None
|
||||
|
||||
@@ -301,6 +301,7 @@ class OrchestratorBase:
|
||||
_transfer_emitter: Any = None
|
||||
_prom_metrics: Any = None
|
||||
_stat_logger: OmniPrometheusStatLogger | None = None
|
||||
_transfer_release_tasks: set[asyncio.Task] = set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -348,6 +349,8 @@ class OrchestratorBase:
|
||||
self._pd_bootstrap_addr = pd_config.get("bootstrap_addr")
|
||||
self._pd_prefill_engine_id = pd_config.get("prefill_engine_id")
|
||||
self.request_states: dict[str, OrchestratorRequestState] = {}
|
||||
# Strong refs for in-flight releases; the loop only weak-refs tasks, so dropping these risks mid-flight GC.
|
||||
self._transfer_release_tasks: set[asyncio.Task] = set()
|
||||
self._init_metrics_state(
|
||||
stage_pools,
|
||||
running_counter,
|
||||
@@ -690,6 +693,36 @@ class OrchestratorBase:
|
||||
output_msg.finished = index == last_index_by_req[output_msg.request_id]
|
||||
return abort_outputs
|
||||
|
||||
def _release_stage_transfer_resources(self, request_ids: list[str]) -> None:
|
||||
"""Drop each stage's inter-stage transfer resources for finished requests.
|
||||
|
||||
This is the only point that knows every stage is done with the request,
|
||||
so it is the only safe place to reclaim segments a consumer never
|
||||
drained. Scheduled rather than awaited: reclaim is best-effort and must
|
||||
not add RPC latency to request teardown.
|
||||
"""
|
||||
if not request_ids:
|
||||
return
|
||||
|
||||
async def _run() -> None:
|
||||
results = await asyncio.gather(
|
||||
*(pool.release_request_resources(request_ids) for pool in self.stage_pools),
|
||||
return_exceptions=True,
|
||||
)
|
||||
for result in results:
|
||||
if isinstance(result, Exception):
|
||||
logger.warning("[Orchestrator] release transfer resources failed: %s", result)
|
||||
|
||||
try:
|
||||
task = asyncio.get_running_loop().create_task(_run())
|
||||
self._transfer_release_tasks.add(task)
|
||||
task.add_done_callback(self._transfer_release_tasks.discard)
|
||||
except RuntimeError:
|
||||
logger.warning(
|
||||
"[Orchestrator] no running event loop; skipped reclaim of transfer resources for %s",
|
||||
request_ids,
|
||||
)
|
||||
|
||||
def _release_request_bindings(self, request_ids: list[str]) -> None:
|
||||
"""Release all stage-local route bindings for the given request ids."""
|
||||
for pool in self.stage_pools:
|
||||
@@ -898,7 +931,7 @@ class OrchestratorBase:
|
||||
req_state is None
|
||||
or req_state.upstream_first_audio
|
||||
or req_state.pending_upstream_first_audio is not None
|
||||
or stage_id + 1 > req_state.final_stage_id
|
||||
or (stage_id if final_output else stage_id + 1) > req_state.final_stage_id
|
||||
):
|
||||
continue
|
||||
audio = mm.get("model_outputs")
|
||||
@@ -935,7 +968,7 @@ class OrchestratorBase:
|
||||
if pending is None or self.request_states.get(req_state.request_id) is not req_state:
|
||||
return
|
||||
source_stage, first_output = pending
|
||||
codec_stage = source_stage + 1
|
||||
codec_stage = source_stage if self.stage_pools[source_stage].final_output else source_stage + 1
|
||||
pool = self.stage_pools[codec_stage]
|
||||
replica_id = pool.get_bound_replica_id(req_state.request_id)
|
||||
if replica_id is None or req_state.request_id not in pool.output_processor.request_states:
|
||||
@@ -1645,6 +1678,7 @@ class OrchestratorBase:
|
||||
if abort:
|
||||
abort_outputs = await self._abort_request_ids(cleanup_ids)
|
||||
self._release_request_bindings(cleanup_ids)
|
||||
self._release_stage_transfer_resources(cleanup_ids)
|
||||
for request_id in cleanup_ids:
|
||||
self._pd_kv_params.pop(request_id, None)
|
||||
req_state = self.request_states.pop(request_id, None)
|
||||
|
||||
@@ -81,19 +81,36 @@ def _signal_exit_code(signum: int) -> int:
|
||||
|
||||
|
||||
def _bind_first_audio_sink(model_executor: Any, output_queue: Any, scheduler: Any) -> bool:
|
||||
"""Let a TP1 in-process runner that decodes first audio deliver it as its own output."""
|
||||
"""Bind in-process first audio to the generic engine output sink."""
|
||||
if not isinstance(model_executor, UniProcExecutor):
|
||||
return False
|
||||
parallel_config = model_executor.vllm_config.parallel_config
|
||||
if parallel_config.tensor_parallel_size != 1 or parallel_config.pipeline_parallel_size != 1:
|
||||
return False
|
||||
worker = getattr(getattr(model_executor, "driver_worker", None), "worker", None)
|
||||
model_runner = getattr(worker, "model_runner", None)
|
||||
model_state = getattr(model_runner, "model_state", None)
|
||||
model = getattr(model_runner, "model", None)
|
||||
if getattr(model, "first_frame_decoder", None) is None or not hasattr(model_state, "set_first_audio_sink"):
|
||||
return False
|
||||
decodes_audio = getattr(model, "first_frame_decoder", None) is not None or (
|
||||
getattr(model, "stream_decoder", None) is not None and bool(getattr(model, "stream_first_audio", False))
|
||||
)
|
||||
from vllm_omni.worker_v2.first_audio_sender import engine_output_queue_sink
|
||||
|
||||
assert model_state is not None
|
||||
model_state.set_first_audio_sink(engine_output_queue_sink(output_queue, scheduler))
|
||||
if decodes_audio and hasattr(model_state, "set_first_audio_sink"):
|
||||
assert model_state is not None
|
||||
model_state.set_first_audio_sink(engine_output_queue_sink(output_queue, scheduler))
|
||||
return True
|
||||
data_plane = getattr(model_runner, "_omni_data_plane", None)
|
||||
get_model = getattr(model_runner, "get_model", None)
|
||||
if data_plane is None or not callable(get_model):
|
||||
return False
|
||||
bind = getattr(get_model(), "bind_first_chunk_fast_path", None)
|
||||
if not callable(bind):
|
||||
return False
|
||||
hook = bind(engine_output_queue_sink(output_queue, scheduler, upstream_first_audio=False))
|
||||
if hook is None:
|
||||
return False
|
||||
data_plane.set_first_chunk_hook(hook)
|
||||
return True
|
||||
|
||||
|
||||
@@ -129,6 +146,21 @@ class StageEngineCoreProc(EngineCoreProc):
|
||||
if _bind_native_data_plane_ready_sink(self.model_executor, self.scheduler):
|
||||
logger.info("Bound native MRv2 connector readiness directly to the scheduler inbox.")
|
||||
|
||||
def omni_release_request_resources(self, request_ids: list[str]) -> None:
|
||||
"""Release this stage's inter-stage transfer resources for *request_ids*.
|
||||
|
||||
Invoked over the UTILITY channel by the orchestrator once every stage
|
||||
has finished with the request. Idempotent and safe for unknown ids.
|
||||
"""
|
||||
adapter = getattr(getattr(self, "scheduler", None), "chunk_transfer_adapter", None)
|
||||
if adapter is None:
|
||||
return
|
||||
for request_id in request_ids or ():
|
||||
try:
|
||||
adapter.release_shm_resources(request_id)
|
||||
except Exception as e:
|
||||
logger.debug("omni_release_request_resources(%s) failed: %s", request_id, e)
|
||||
|
||||
def preprocess_add_request(self, request: OmniEngineCoreRequest) -> tuple[Any, int]:
|
||||
"""Preserve omni payloads when vLLM builds its scheduler request."""
|
||||
scheduler_request, current_wave = super().preprocess_add_request(request)
|
||||
|
||||
@@ -99,6 +99,8 @@ class StagePool:
|
||||
|
||||
DISPATCH_WAIT_TIMEOUT_S: float = 10.0
|
||||
DISPATCH_RETRY_INTERVAL_S: float = 0.1
|
||||
# A replica that dies mid-release never answers; without a bound its background release never finishes.
|
||||
RELEASE_RPC_TIMEOUT_S: float = 5.0
|
||||
# Only these EngineCore helpers may skip collective_rpc_async. A generic
|
||||
# ``{method}_async`` on AsyncMPClient must not silently drop timeout.
|
||||
_CACHE_RESET_METHODS = frozenset({"reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"})
|
||||
@@ -136,6 +138,9 @@ class StagePool:
|
||||
self.clients: list[StagePoolClient | None] = list(normalized_clients)
|
||||
self._output_processor = output_processor
|
||||
self._stage_vllm_config = stage_vllm_config
|
||||
self._has_chunk_transfer_adapter = bool(
|
||||
getattr(getattr(stage_vllm_config, "model_config", None), "async_chunk", False)
|
||||
)
|
||||
self._next_replica_id = 0
|
||||
self._request_bindings: dict[str, int] = {}
|
||||
self._unavailable_replicas: set[int] = set()
|
||||
@@ -1305,6 +1310,28 @@ class StagePool:
|
||||
|
||||
return abort_outputs
|
||||
|
||||
async def release_request_resources(self, request_ids: list[str]) -> None:
|
||||
"""Ask every live replica to drop transfer resources for *request_ids*.
|
||||
|
||||
Broadcast rather than binding-routed: the orchestrator releases route
|
||||
bindings as part of the same teardown, so a binding lookup here would
|
||||
race it. The engine-core handler is idempotent for unknown ids.
|
||||
"""
|
||||
if not request_ids or not self._has_chunk_transfer_adapter:
|
||||
return
|
||||
ids = list(request_ids)
|
||||
|
||||
async def release(replica_id: int, call: Any) -> None:
|
||||
try:
|
||||
await asyncio.wait_for(call("omni_release_request_resources", ids), timeout=self.RELEASE_RPC_TIMEOUT_S)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"[StagePool-%s] release_request_resources on replica %s failed: %r", self.stage_id, replica_id, e
|
||||
)
|
||||
|
||||
calls = [(i, getattr(self.clients[i], "call_utility_async", None)) for i in self.live_replica_ids()]
|
||||
await asyncio.gather(*(release(i, call) for i, call in calls if call is not None))
|
||||
|
||||
async def collective_rpc(
|
||||
self,
|
||||
replica_id: int,
|
||||
|
||||
Reference in new issue
Block a user