Files
vllm-omni/vllm_omni/worker_v2/omni_model_runner.py
T
Taichang ZhouandClaude Opus 4.8 9678ae58cf [MR V2] Migrate to Model Runner V2
Squashed migration rebased onto current main. Collapses the 46-commit
dev/migrate-MR-v2 history (10 merge commits + superseded pre-rebase work)
into a single commit, so the diff reflects only the genuine MR-V2 feature
(worker_v2/ runners + model_states, omni schedulers, stage engine pipeline,
pooling_output serialization bridge). Content already merged to main
separately (step_audio2, tts_adapters, npu/_310p, etc.) drops out.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-20 07:19:40 +00:00

394 lines
17 KiB
Python

"""Omni v2 GPU model runner hooks."""
from __future__ import annotations
import threading
from typing import Any
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import set_forward_context
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.worker.gpu.model_runner import (
BatchDescriptor,
BatchExecutionDescriptor,
ExecuteModelState,
GPUModelRunner,
IntermediateTensors,
build_slot_mappings_by_layer,
get_uniform_token_count,
)
from vllm_omni.compat import make_filtered_namedtuple
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.worker_v2.forward_compat import add_forward_compat_kwargs
from vllm_omni.worker_v2.model_states import init_omni_model_state
from vllm_omni.worker_v2.model_states.intermediate_buffer import (
_resolve_additional_information,
)
from vllm_omni.worker_v2.model_states.omni_model_state import OmniModelState
logger = init_logger(__name__)
_model_state_patch_lock = threading.RLock()
_GPU_MODEL_RUNNER_HAS_SHUTDOWN = hasattr(GPUModelRunner, "shutdown")
# Fields the omni runners still pass to ExecuteModelState that current vLLM no
# longer declares on the namedtuple. They are filtered out by
# make_filtered_namedtuple and carried out-of-band instead:
# - num_tokens_across_dp: only consumed inside execute_model's forward context.
# - kv_connector_output: removed from ExecuteModelState in vLLM 0.23.0 (post_forward
# now runs in sample_tokens). Carried via self._last_kv_connector_output (AR) /
# self._gen_kv_connector_output (generation).
_KNOWN_EXECUTE_MODEL_STATE_COMPAT_FIELDS = {"num_tokens_across_dp", "kv_connector_output"}
def _make_execute_model_state(**kwargs):
state, unknown = make_filtered_namedtuple(
ExecuteModelState,
known_extra_fields=_KNOWN_EXECUTE_MODEL_STATE_COMPAT_FIELDS,
**kwargs,
)
if unknown:
logger.warning("Unknown fields passed to ExecuteModelState: %s", sorted(unknown))
return state
def _needs_capture_tensor_unwrap(model: Any) -> bool:
return bool(getattr(model, "_returns_tuple", False) or getattr(model, "model_stage", None) == "thinker")
class OmniGPUModelRunner(GPUModelRunner):
"""Thin layer over v2 ``GPUModelRunner`` for Omni lifecycle hooks."""
model_state: OmniModelState
_last_aux_output: Any
_last_multimodal_outputs: dict[str, Any] | None
_model_returns_tuple: bool
def shutdown(self) -> None:
if _GPU_MODEL_RUNNER_HAS_SHUTDOWN:
super().shutdown()
def load_model(self, *args: Any, **kwargs: Any) -> None:
import vllm.v1.worker.gpu.model_runner as _mr_module
with _model_state_patch_lock:
_orig = _mr_module.init_model_state
_mr_module.init_model_state = init_omni_model_state
try:
super().load_model(*args, **kwargs)
finally:
_mr_module.init_model_state = _orig
self._last_aux_output = None
self._last_multimodal_outputs = None
self._last_kv_connector_output = None
self._model_returns_tuple = _needs_capture_tensor_unwrap(self.model)
self._exclude_full_graph = self._model_returns_tuple or hasattr(self.model, "_last_captured_layers")
# Preprocess models own embedding buffers; encoder_runner sizing would mismatch.
if getattr(self.model, "has_preprocess", False) and self.supports_mm_inputs:
self.supports_mm_inputs = False
self.encoder_cache = None
# ------------------------------------------------------------------
# CUDA Graph: conditionally exclude FULL mode
# ------------------------------------------------------------------
def capture_model(self) -> int:
"""Handle CUDA graph capture for Omni models.
Exclude FULL mode for tuple-returning models because
``run_fullgraph`` bypasses Python-level tuple intercept.
For PIECEWISE capture, the warmup pass runs with
``CUDAGraphMode.NONE`` which hits ``torch.empty_like(hidden_states)``
in the cudagraph framework. If the model returns a tuple, that call
crashes. We temporarily wrap the model's forward to extract only the
tensor part during capture, then restore the original forward.
"""
if self._exclude_full_graph:
mgr = self.cudagraph_manager
# NOTE: ``_capture_descs`` / ``_candidates`` are private CudaGraphManager
# internals (present and same-typed as of vLLM 0.23.0). Guard with hasattr
# so a future vLLM that renames/removes them degrades to a warning instead
# of an AttributeError; the worst case is FULL graphs not being excluded.
if hasattr(mgr, "_capture_descs") and hasattr(mgr, "_candidates"):
if CUDAGraphMode.FULL in mgr._capture_descs:
del mgr._capture_descs[CUDAGraphMode.FULL]
for i, descs in enumerate(mgr._candidates):
mgr._candidates[i] = [d for d in descs if d.cg_mode != CUDAGraphMode.FULL]
logger.info("Excluded FULL CUDA graph capture for Omni model. PIECEWISE graphs will still be captured.")
else:
logger.warning(
"CudaGraphManager lacks _capture_descs/_candidates; cannot exclude "
"FULL graph capture for tuple-returning Omni model. vLLM internals "
"may have changed — verify omni_model_runner.capture_model()."
)
# Wrap model forward during capture so tuple returns don't crash
# torch.empty_like() in the PIECEWISE warmup pass.
if self._model_returns_tuple:
original_forward = self.model.forward
def _capture_forward(*args: Any, **kwargs: Any) -> torch.Tensor:
output = original_forward(*args, **kwargs)
if isinstance(output, OmniOutput):
return output.text_hidden_states
if isinstance(output, tuple):
return output[0]
return output
self.model.forward = _capture_forward # type: ignore[assignment]
try:
return super().capture_model()
finally:
self.model.forward = original_forward # type: ignore[assignment]
return super().capture_model()
def _dispatch_batch_descriptor(
self,
*,
num_reqs: int,
num_toks: int,
uniform_tok_count: int,
use_eager: bool,
):
if use_eager:
batch_desc = BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=num_toks,
num_reqs=num_reqs,
)
else:
batch_desc = self.cudagraph_manager.dispatch(num_reqs, num_toks, uniform_tok_count)
if self.dp_size > 1:
from vllm.v1.worker.gpu.dp_utils import sync_cudagraph_and_dp_padding
return sync_cudagraph_and_dp_padding(
self.cudagraph_manager,
batch_desc,
num_toks,
num_reqs,
uniform_tok_count,
self.dp_size,
self.dp_rank,
)
return batch_desc, None
@torch.inference_mode()
def execute_model(
self,
scheduler_output: SchedulerOutput,
intermediate_tensors: IntermediateTensors | None = None,
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
is_profile: bool = False,
) -> Any:
if not dummy_run:
self.finish_requests(scheduler_output)
self.free_states(scheduler_output)
self.add_requests(scheduler_output)
self.update_requests(scheduler_output)
self.block_tables.apply_staged_writes()
if scheduler_output.total_num_scheduled_tokens == 0:
return self.kv_connector.no_forward(scheduler_output)
num_reqs = len(scheduler_output.num_scheduled_tokens)
num_toks = scheduler_output.total_num_scheduled_tokens
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
uniform_tok_count = get_uniform_token_count(num_reqs, num_toks, max_query_len)
# Encoder-decoder models: disable compilation when encoder inputs
# are scheduled (dynamic cross-attention cache updates).
skip_compiled = self.is_encoder_decoder and bool(scheduler_output.scheduled_encoder_inputs)
batch_desc, num_tokens_across_dp = self._dispatch_batch_descriptor(
num_reqs=num_reqs,
num_toks=num_toks,
uniform_tok_count=uniform_tok_count,
use_eager=is_profile or skip_compiled,
)
if batch_desc.num_tokens == 0:
return self.kv_connector.no_forward(scheduler_output)
if not dummy_run:
input_batch = self.prepare_inputs(scheduler_output, batch_desc)
block_tables, slot_mappings = self.prepare_attn(input_batch)
if self.lora_config:
lora_inputs = self.lora_state.make_lora_inputs(
input_batch.req_ids,
input_batch.idx_mapping_np,
input_batch.num_scheduled_tokens,
)
self._set_active_loras(*lora_inputs)
else:
from vllm.v1.worker.gpu.input_batch import InputBatch
input_batch = InputBatch.make_dummy(
batch_desc.num_reqs or num_reqs,
batch_desc.num_tokens,
self.input_buffers,
)
if not skip_attn_for_dummy_run:
block_tables, slot_mappings = self.prepare_dummy_attn(input_batch)
else:
block_tables = None
slot_mappings = None
attn_metadata = None
slot_mappings_by_layer = None
if not (dummy_run and skip_attn_for_dummy_run):
assert slot_mappings is not None
slot_mappings_by_layer = build_slot_mappings_by_layer(slot_mappings, self.kv_cache_config)
assert block_tables is not None
attn_metadata = self.model_state.prepare_attn(
input_batch,
batch_desc.cg_mode,
block_tables,
slot_mappings,
self.attn_groups,
self.kv_cache_config,
)
inputs_embeds = None
if self.supports_mm_inputs and self.is_first_pp_rank:
# vLLM 0.23 dropped the req_states arg from get_mm_embeddings.
inputs_embeds = self.model_state.get_mm_embeddings(
scheduler_output.scheduled_encoder_inputs,
input_batch,
)
model_inputs: dict[str, Any] = {
"input_ids": input_batch.input_ids,
"positions": input_batch.positions,
"inputs_embeds": inputs_embeds,
"intermediate_tensors": intermediate_tensors,
**self.model_state.prepare_inputs(input_batch, self.req_states),
}
add_forward_compat_kwargs(model_inputs, input_batch, self.sampler)
if not self.is_first_pp_rank:
model_inputs["input_ids"] = None
model_inputs["inputs_embeds"] = None
assert intermediate_tensors is not None
# ★ PRE-FORWARD: per-request preprocess + batched MTP.
# Runs for ALL graph modes (FULL, PIECEWISE, NONE).
# For FULL graph: OmniModelState provides a static inputs_embeds
# buffer that was captured by the graph. Preprocess writes
# in-place to this buffer, and FULL graph replay reads the
# updated values from the same tensor address.
if not dummy_run:
self.model_state.run_preprocess(input_batch, model_inputs)
# --- Model forward ---
if batch_desc.cg_mode == CUDAGraphMode.FULL:
# FULL graph replay. Preprocess already wrote to the static
# inputs_embeds buffer above.
assert self.cudagraph_manager is not None
self.kv_connector.pre_forward(scheduler_output)
model_output = self.cudagraph_manager.run_fullgraph(batch_desc)
hidden_states = model_output
self._last_aux_output = None
self._last_multimodal_outputs = None
else:
batch_descriptor = BatchDescriptor(
num_tokens=input_batch.num_tokens_after_padding,
has_lora=self.lora_config is not None,
)
with set_forward_context(
attn_metadata,
self.vllm_config,
num_tokens=input_batch.num_tokens_after_padding,
cudagraph_runtime_mode=batch_desc.cg_mode,
num_tokens_across_dp=num_tokens_across_dp,
batch_descriptor=batch_descriptor,
slot_mapping=slot_mappings_by_layer,
skip_compiled=skip_compiled,
):
self.kv_connector.pre_forward(scheduler_output)
model_output = self.model(**model_inputs)
# Extract hidden_states from model output.
self._last_aux_output = None
self._last_multimodal_outputs = None
if isinstance(model_output, OmniOutput):
hidden_states = model_output.text_hidden_states
if model_output.multimodal_outputs:
self._last_multimodal_outputs = model_output.multimodal_outputs
elif isinstance(model_output, tuple) and len(model_output) == 2:
hidden_states, self._last_aux_output = model_output
if hasattr(self.model, "_last_captured_layers"):
self.model._last_captured_layers = self._last_aux_output
else:
hidden_states = model_output
if not dummy_run and isinstance(hidden_states, torch.Tensor):
self.model_state.run_postprocess(hidden_states, input_batch)
kv_connector_output = self.kv_connector.post_forward(scheduler_output.finished_req_ids)
self._last_kv_connector_output = kv_connector_output
self.execute_model_state = _make_execute_model_state(
input_batch=input_batch,
attn_metadata=attn_metadata,
slot_mappings_by_layer=slot_mappings_by_layer,
hidden_states=hidden_states,
aux_hidden_states=None,
finished_req_ids=scheduler_output.finished_req_ids,
kv_connector_output=kv_connector_output,
num_tokens_across_dp=num_tokens_across_dp,
)
if not self.is_last_pp_rank:
assert isinstance(hidden_states, IntermediateTensors)
hidden_states.kv_connector_output = kv_connector_output
return hidden_states
assert isinstance(hidden_states, torch.Tensor)
return None
# ------------------------------------------------------------------
# Request lifecycle: update intermediate buffer from cached requests
# ------------------------------------------------------------------
def update_requests(self, scheduler_output: SchedulerOutput) -> None:
"""Merge updated additional_information into intermediate_buffer.
In async_chunk mode, chunk_transfer_adapter attaches updated
additional_information (e.g. thinker_decode_embeddings) to
OmniCachedRequestData for cached requests every schedule step.
Upstream GPUModelRunner.update_requests does not handle this
field, so we merge it into the intermediate buffer here.
"""
super().update_requests(scheduler_output)
cached = scheduler_output.scheduled_cached_reqs
addl_info = getattr(cached, "additional_information", None)
if not addl_info:
return
for req_id, info in addl_info.items():
if info is None:
continue
req_idx = self.req_states.req_id_to_index.get(req_id)
if req_idx is None:
continue
resolved = _resolve_additional_information(info)
if resolved:
self.model_state.intermediate_buffer.update(req_idx, resolved)
# ------------------------------------------------------------------
# Request lifecycle: clean up intermediate buffer on finish
# ------------------------------------------------------------------
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
# IMPORTANT: Must query req_id_to_index BEFORE super().finish_requests()
# because super() calls req_states.remove_request(req_id) which pops the
# mapping and returns the slot index to free_indices.
finished = scheduler_output.finished_req_ids
preempted = scheduler_output.preempted_req_ids
all_done = finished | preempted if preempted else finished
for req_id in all_done:
idx = self.req_states.req_id_to_index.get(req_id)
if idx is not None:
self.model_state.remove_request(idx)
super().finish_requests(scheduler_output)