[MR V2] Migrate to Model Runner V2

Squashed migration rebased onto current main. Collapses the 46-commit
dev/migrate-MR-v2 history (10 merge commits + superseded pre-rebase work)
into a single commit, so the diff reflects only the genuine MR-V2 feature
(worker_v2/ runners + model_states, omni schedulers, stage engine pipeline,
pooling_output serialization bridge). Content already merged to main
separately (step_audio2, tts_adapters, npu/_310p, etc.) drops out.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Taichang ZhouandClaude Opus 4.8 committed 2026-06-20 07:19:40 +00:00
1 parent cead615288
commit 9678ae58cf
74 files changed
+8244 -95

No files matched your search

@@ -43,7 +43,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
"detokenize": True,
"stop_token_ids": [8294],
"stop_token_ids": [8292, 8294],
},
),
StagePipelineConfig(
@@ -320,6 +320,12 @@ class Qwen3OmniMoeForConditionalGeneration(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec] | None = None,
# V1 runner passes the full multimodal RoPE inputs explicitly, while
# vLLM 0.20 calls this method with only input_tokens/mm_features.
hf_config: object | None = None,
image_grid_thw: list[list[int]] | torch.Tensor | None = None,
video_grid_thw: list[list[int]] | torch.Tensor | None = None,
second_per_grid_ts: list[float] | None = None,
**kwargs: object,
) -> tuple[torch.Tensor, int]:
if self.model_stage == "thinker":
@@ -30,7 +30,6 @@ from typing import Any, cast
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from packaging.version import Version
from transformers import PretrainedConfig
from transformers import __version__ as TRANSFORMERS_VERSION
@@ -163,66 +162,42 @@ class Qwen3Omni_VisionTransformer(_Qwen3Omni_VisionTransformer):
rotary_pos_emb_cos, rotary_pos_emb_sin = self.rot_pos_emb(grid_thw)
if isinstance(grid_thw, torch.Tensor):
grid_thw_tensor = grid_thw.to(self.device)
grid_thw_np = grid_thw.cpu().numpy().astype(np.int32)
else:
grid_thw_tensor = torch.as_tensor(grid_thw, dtype=torch.int32, device=self.device)
grid_thw_np = np.array(grid_thw, dtype=np.int32)
try:
cu_seqlens = torch.repeat_interleave(
grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2],
grid_thw_tensor[:, 0],
).cumsum(
dim=0,
dtype=grid_thw_tensor.dtype if torch.jit.is_tracing() else torch.int32,
)
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
except RuntimeError:
logger.warning(
"torch.repeat_interleave not executable, switching to vectorized searchsorted implementation."
)
repeat_counts = grid_thw_tensor[:, 0]
values = grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2]
repeat_cumsum = repeat_counts.cumsum(0)
total_items = repeat_cumsum[-1].item()
indices = torch.searchsorted(
repeat_cumsum,
torch.arange(total_items, device=grid_thw_tensor.device),
right=True,
)
cu_seqlens = values[indices].cumsum(
dim=0,
dtype=grid_thw_tensor.dtype if torch.jit.is_tracing() else torch.int32,
)
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
cu_seqlens = np.repeat(grid_thw_np[:, 1] * grid_thw_np[:, 2], grid_thw_np[:, 0]).cumsum(axis=0, dtype=np.int32)
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
hidden_states = hidden_states.unsqueeze(1)
rotary_pos_emb_cos = rotary_pos_emb_cos.to(hidden_states.device)
rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device)
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
grid_thw_np = grid_thw_tensor.cpu().numpy().astype(np.int32)
cu_seqlens_np = np.repeat(grid_thw_np[:, 1] * grid_thw_np[:, 2], grid_thw_np[:, 0]).cumsum(
axis=0, dtype=np.int32
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(self.attn_backend, cu_seqlens, self.device)
max_seqlen = torch.tensor(
MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens),
dtype=torch.int32,
device=self.device,
)
cu_seqlens_np = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens_np])
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
self.attn_backend,
cu_seqlens_np,
cu_seqlens,
self.hidden_size,
self.tp_size,
self.device,
)
hidden_states = hidden_states.unsqueeze(1)
hidden_states_list = []
deepstack_visual_indexes = self.deepstack_visual_indexes
for layer_num, blk in enumerate(self.blocks):
hidden_states = blk(
hidden_states,
hidden_states = hidden_states + blk.attn(
blk.norm1(hidden_states),
cu_seqlens=cu_seqlens,
rotary_pos_emb_cos=rotary_pos_emb_cos,
rotary_pos_emb_sin=rotary_pos_emb_sin,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
hidden_states = hidden_states + blk.mlp(blk.norm2(hidden_states))
if deepstack_visual_indexes is not None and layer_num in deepstack_visual_indexes:
hidden_states_list.append(hidden_states)
@@ -784,13 +759,19 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
mm_item_counts,
)
else:
if use_audio_in_video and "audio" in mm_prompt_updates:
if use_audio_in_video:
# When use_audio_in_video=True, audio is extracted from video and embedded
# in the video placeholder tokens. We should:
# 1. Filter out audio from prompt updates (audio has no separate placeholder)
# 2. Apply remaining updates (video, image, etc.)
# 3. Derive audio placeholders from video placeholders
filtered_updates = {k: v for k, v in mm_prompt_updates.items() if k != "audio"}
prompt_ids, mm_placeholders = self._apply_prompt_updates(
prompt_ids,
filtered_updates,
)
mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates)
# Derive audio placeholders from video placeholders
mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_item_counts)
else:
prompt_ids, mm_placeholders = self._apply_prompt_updates(
prompt_ids,
@@ -909,7 +890,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
nonlocal audio_in_video_item_idx
audio_num_features = audio_output_lengths[audio_in_video_item_idx]
audio_num_features = audio_output_lengths[audio_in_video_item_idx + item_idx]
video_grid_thw = out_mm_data["video_grid_thw"][item_idx]
audio_in_video_item_idx += 1
@@ -955,17 +936,27 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
def _derive_audio_from_video_placeholders(
self,
placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
mm_prompt_updates: MultiModalPromptUpdates,
mm_item_counts: Mapping[str, int],
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
"""
Helper to derive audio placeholders from video placeholders when
use_audio_in_video=True.
In use_audio_in_video mode, audio is extracted from video and embedded
within the video placeholder tokens. This function creates audio placeholder
info by extracting the audio token positions from video placeholders.
Args:
placeholders: Current placeholders (should contain "video")
mm_item_counts: Counts of multimodal items from mm_items.get_all_counts()
"""
if "video" not in placeholders:
return placeholders
# Validate audio and video counts match
# In use_audio_in_video mode, audio count comes from mm_items (extracted from video)
num_videos = len(placeholders["video"])
num_audios = len(mm_prompt_updates.get("audio", []))
num_audios = mm_item_counts.get("audio", 0)
if num_audios != num_videos:
raise ValueError(
f"use_audio_in_video requires equal number of audio and video items, got {num_audios=}, {num_videos=}"
@@ -1285,7 +1276,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
return []
# The result multimodal_embeddings is tuple of tensors, with each
# tensor corresponding to a multimodal data item (image or video).
# tensor correspoending to a multimodal data item (image or video).
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
# NOTE: It is important to iterate over the keys in this dictionary
@@ -1309,6 +1300,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
inputs_embeds = self._embed_text_input_ids(
input_ids,
@@ -536,6 +536,10 @@ class Qwen3TTSConfig(PretrainedConfig):
if any("Code2Wav" in str(a) for a in archs):
if hasattr(config, "rope_parameters"):
delattr(config, "rope_parameters")
# Code2Wav doesn't use position embeddings, but vLLM validates
# max_model_len <= max_position_embeddings. Set a large value
# to allow the pipeline.yaml max_model_len (65536) to pass validation.
config.max_position_embeddings = 131072
return config