Merge main into dev/vllm-align for vLLM 0.31 release

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-10-03 21:05:17 +08:00
commit 008fa25263
252 files changed
+27589 -1237

No files matched your search

+3
View File
@@ -198,6 +198,7 @@ class OmniEngineArgs(EngineArgs):
async_chunk: bool = False
session_mode: str = "turn"
retains_state_across_chunks: bool = False
supports_running_prefix_cache_reset: bool = True
use_v2_model_runner: bool = False
supports_native_mrv2_data_plane: bool = False
# WS-A: Stage-1 active stream slots. 0 = legacy preempt-everything.
@@ -432,6 +433,7 @@ class OmniEngineArgs(EngineArgs):
async_chunk=self.async_chunk,
session_mode=self.session_mode,
retains_state_across_chunks=self.retains_state_across_chunks,
supports_running_prefix_cache_reset=self.supports_running_prefix_cache_reset,
use_v2_model_runner=self.use_v2_model_runner,
supports_native_mrv2_data_plane=self.supports_native_mrv2_data_plane,
active_stream_window=self.active_stream_window,
@@ -560,6 +562,7 @@ class OrchestratorArgs:
# === Diffusion model config ===
num_gpus: int | None = None
model_class_name: str | None = None
hsdp_weight_load_strategy: str | None = None
diffusion_load_format: str | None = None
lora_path: list[str] | None = None
lora_backend: str | None = None
+36 -2
View File
@@ -301,6 +301,7 @@ class OrchestratorBase:
_transfer_emitter: Any = None
_prom_metrics: Any = None
_stat_logger: OmniPrometheusStatLogger | None = None
_transfer_release_tasks: set[asyncio.Task] = set()
def __init__(
self,
@@ -348,6 +349,8 @@ class OrchestratorBase:
self._pd_bootstrap_addr = pd_config.get("bootstrap_addr")
self._pd_prefill_engine_id = pd_config.get("prefill_engine_id")
self.request_states: dict[str, OrchestratorRequestState] = {}
# Strong refs for in-flight releases; the loop only weak-refs tasks, so dropping these risks mid-flight GC.
self._transfer_release_tasks: set[asyncio.Task] = set()
self._init_metrics_state(
stage_pools,
running_counter,
@@ -690,6 +693,36 @@ class OrchestratorBase:
output_msg.finished = index == last_index_by_req[output_msg.request_id]
return abort_outputs
def _release_stage_transfer_resources(self, request_ids: list[str]) -> None:
"""Drop each stage's inter-stage transfer resources for finished requests.
This is the only point that knows every stage is done with the request,
so it is the only safe place to reclaim segments a consumer never
drained. Scheduled rather than awaited: reclaim is best-effort and must
not add RPC latency to request teardown.
"""
if not request_ids:
return
async def _run() -> None:
results = await asyncio.gather(
*(pool.release_request_resources(request_ids) for pool in self.stage_pools),
return_exceptions=True,
)
for result in results:
if isinstance(result, Exception):
logger.warning("[Orchestrator] release transfer resources failed: %s", result)
try:
task = asyncio.get_running_loop().create_task(_run())
self._transfer_release_tasks.add(task)
task.add_done_callback(self._transfer_release_tasks.discard)
except RuntimeError:
logger.warning(
"[Orchestrator] no running event loop; skipped reclaim of transfer resources for %s",
request_ids,
)
def _release_request_bindings(self, request_ids: list[str]) -> None:
"""Release all stage-local route bindings for the given request ids."""
for pool in self.stage_pools:
@@ -898,7 +931,7 @@ class OrchestratorBase:
req_state is None
or req_state.upstream_first_audio
or req_state.pending_upstream_first_audio is not None
or stage_id + 1 > req_state.final_stage_id
or (stage_id if final_output else stage_id + 1) > req_state.final_stage_id
):
continue
audio = mm.get("model_outputs")
@@ -935,7 +968,7 @@ class OrchestratorBase:
if pending is None or self.request_states.get(req_state.request_id) is not req_state:
return
source_stage, first_output = pending
codec_stage = source_stage + 1
codec_stage = source_stage if self.stage_pools[source_stage].final_output else source_stage + 1
pool = self.stage_pools[codec_stage]
replica_id = pool.get_bound_replica_id(req_state.request_id)
if replica_id is None or req_state.request_id not in pool.output_processor.request_states:
@@ -1645,6 +1678,7 @@ class OrchestratorBase:
if abort:
abort_outputs = await self._abort_request_ids(cleanup_ids)
self._release_request_bindings(cleanup_ids)
self._release_stage_transfer_resources(cleanup_ids)
for request_id in cleanup_ids:
self._pd_kv_params.pop(request_id, None)
req_state = self.request_states.pop(request_id, None)
+37 -5
View File
@@ -81,19 +81,36 @@ def _signal_exit_code(signum: int) -> int:
def _bind_first_audio_sink(model_executor: Any, output_queue: Any, scheduler: Any) -> bool:
"""Let a TP1 in-process runner that decodes first audio deliver it as its own output."""
"""Bind in-process first audio to the generic engine output sink."""
if not isinstance(model_executor, UniProcExecutor):
return False
parallel_config = model_executor.vllm_config.parallel_config
if parallel_config.tensor_parallel_size != 1 or parallel_config.pipeline_parallel_size != 1:
return False
worker = getattr(getattr(model_executor, "driver_worker", None), "worker", None)
model_runner = getattr(worker, "model_runner", None)
model_state = getattr(model_runner, "model_state", None)
model = getattr(model_runner, "model", None)
if getattr(model, "first_frame_decoder", None) is None or not hasattr(model_state, "set_first_audio_sink"):
return False
decodes_audio = getattr(model, "first_frame_decoder", None) is not None or (
getattr(model, "stream_decoder", None) is not None and bool(getattr(model, "stream_first_audio", False))
)
from vllm_omni.worker_v2.first_audio_sender import engine_output_queue_sink
assert model_state is not None
model_state.set_first_audio_sink(engine_output_queue_sink(output_queue, scheduler))
if decodes_audio and hasattr(model_state, "set_first_audio_sink"):
assert model_state is not None
model_state.set_first_audio_sink(engine_output_queue_sink(output_queue, scheduler))
return True
data_plane = getattr(model_runner, "_omni_data_plane", None)
get_model = getattr(model_runner, "get_model", None)
if data_plane is None or not callable(get_model):
return False
bind = getattr(get_model(), "bind_first_chunk_fast_path", None)
if not callable(bind):
return False
hook = bind(engine_output_queue_sink(output_queue, scheduler, upstream_first_audio=False))
if hook is None:
return False
data_plane.set_first_chunk_hook(hook)
return True
@@ -129,6 +146,21 @@ class StageEngineCoreProc(EngineCoreProc):
if _bind_native_data_plane_ready_sink(self.model_executor, self.scheduler):
logger.info("Bound native MRv2 connector readiness directly to the scheduler inbox.")
def omni_release_request_resources(self, request_ids: list[str]) -> None:
"""Release this stage's inter-stage transfer resources for *request_ids*.
Invoked over the UTILITY channel by the orchestrator once every stage
has finished with the request. Idempotent and safe for unknown ids.
"""
adapter = getattr(getattr(self, "scheduler", None), "chunk_transfer_adapter", None)
if adapter is None:
return
for request_id in request_ids or ():
try:
adapter.release_shm_resources(request_id)
except Exception as e:
logger.debug("omni_release_request_resources(%s) failed: %s", request_id, e)
def preprocess_add_request(self, request: OmniEngineCoreRequest) -> tuple[Any, int]:
"""Preserve omni payloads when vLLM builds its scheduler request."""
scheduler_request, current_wave = super().preprocess_add_request(request)
+27
View File
@@ -99,6 +99,8 @@ class StagePool:
DISPATCH_WAIT_TIMEOUT_S: float = 10.0
DISPATCH_RETRY_INTERVAL_S: float = 0.1
# A replica that dies mid-release never answers; without a bound its background release never finishes.
RELEASE_RPC_TIMEOUT_S: float = 5.0
# Only these EngineCore helpers may skip collective_rpc_async. A generic
# ``{method}_async`` on AsyncMPClient must not silently drop timeout.
_CACHE_RESET_METHODS = frozenset({"reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"})
@@ -136,6 +138,9 @@ class StagePool:
self.clients: list[StagePoolClient | None] = list(normalized_clients)
self._output_processor = output_processor
self._stage_vllm_config = stage_vllm_config
self._has_chunk_transfer_adapter = bool(
getattr(getattr(stage_vllm_config, "model_config", None), "async_chunk", False)
)
self._next_replica_id = 0
self._request_bindings: dict[str, int] = {}
self._unavailable_replicas: set[int] = set()
@@ -1305,6 +1310,28 @@ class StagePool:
return abort_outputs
async def release_request_resources(self, request_ids: list[str]) -> None:
"""Ask every live replica to drop transfer resources for *request_ids*.
Broadcast rather than binding-routed: the orchestrator releases route
bindings as part of the same teardown, so a binding lookup here would
race it. The engine-core handler is idempotent for unknown ids.
"""
if not request_ids or not self._has_chunk_transfer_adapter:
return
ids = list(request_ids)
async def release(replica_id: int, call: Any) -> None:
try:
await asyncio.wait_for(call("omni_release_request_resources", ids), timeout=self.RELEASE_RPC_TIMEOUT_S)
except Exception as e:
logger.warning(
"[StagePool-%s] release_request_resources on replica %s failed: %r", self.stage_id, replica_id, e
)
calls = [(i, getattr(self.clients[i], "call_utility_async", None)) for i in self.live_replica_ids()]
await asyncio.gather(*(release(i, call) for i, call in calls if call is not None))
async def collective_rpc(
self,
replica_id: int,