[MR V2] Migrate to Model Runner V2
Squashed migration rebased onto current main. Collapses the 46-commit dev/migrate-MR-v2 history (10 merge commits + superseded pre-rebase work) into a single commit, so the diff reflects only the genuine MR-V2 feature (worker_v2/ runners + model_states, omni schedulers, stage engine pipeline, pooling_output serialization bridge). Content already merged to main separately (step_audio2, tts_adapters, npu/_310p, etc.) drops out. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cead615288
commit
9678ae58cf
74 files changed
+8244
-95
No files matched your search
@@ -43,7 +43,7 @@ QWEN2_5_OMNI_PIPELINE = PipelineConfig(
|
||||
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": True,
|
||||
"stop_token_ids": [8294],
|
||||
"stop_token_ids": [8292, 8294],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
|
||||
@@ -320,6 +320,12 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[MultiModalFeatureSpec] | None = None,
|
||||
# V1 runner passes the full multimodal RoPE inputs explicitly, while
|
||||
# vLLM 0.20 calls this method with only input_tokens/mm_features.
|
||||
hf_config: object | None = None,
|
||||
image_grid_thw: list[list[int]] | torch.Tensor | None = None,
|
||||
video_grid_thw: list[list[int]] | torch.Tensor | None = None,
|
||||
second_per_grid_ts: list[float] | None = None,
|
||||
**kwargs: object,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
if self.model_stage == "thinker":
|
||||
|
||||
@@ -30,7 +30,6 @@ from typing import Any, cast
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from packaging.version import Version
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||
@@ -163,66 +162,42 @@ class Qwen3Omni_VisionTransformer(_Qwen3Omni_VisionTransformer):
|
||||
rotary_pos_emb_cos, rotary_pos_emb_sin = self.rot_pos_emb(grid_thw)
|
||||
|
||||
if isinstance(grid_thw, torch.Tensor):
|
||||
grid_thw_tensor = grid_thw.to(self.device)
|
||||
grid_thw_np = grid_thw.cpu().numpy().astype(np.int32)
|
||||
else:
|
||||
grid_thw_tensor = torch.as_tensor(grid_thw, dtype=torch.int32, device=self.device)
|
||||
grid_thw_np = np.array(grid_thw, dtype=np.int32)
|
||||
|
||||
try:
|
||||
cu_seqlens = torch.repeat_interleave(
|
||||
grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2],
|
||||
grid_thw_tensor[:, 0],
|
||||
).cumsum(
|
||||
dim=0,
|
||||
dtype=grid_thw_tensor.dtype if torch.jit.is_tracing() else torch.int32,
|
||||
)
|
||||
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
|
||||
except RuntimeError:
|
||||
logger.warning(
|
||||
"torch.repeat_interleave not executable, switching to vectorized searchsorted implementation."
|
||||
)
|
||||
repeat_counts = grid_thw_tensor[:, 0]
|
||||
values = grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2]
|
||||
repeat_cumsum = repeat_counts.cumsum(0)
|
||||
total_items = repeat_cumsum[-1].item()
|
||||
indices = torch.searchsorted(
|
||||
repeat_cumsum,
|
||||
torch.arange(total_items, device=grid_thw_tensor.device),
|
||||
right=True,
|
||||
)
|
||||
cu_seqlens = values[indices].cumsum(
|
||||
dim=0,
|
||||
dtype=grid_thw_tensor.dtype if torch.jit.is_tracing() else torch.int32,
|
||||
)
|
||||
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
|
||||
cu_seqlens = np.repeat(grid_thw_np[:, 1] * grid_thw_np[:, 2], grid_thw_np[:, 0]).cumsum(axis=0, dtype=np.int32)
|
||||
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
|
||||
|
||||
hidden_states = hidden_states.unsqueeze(1)
|
||||
rotary_pos_emb_cos = rotary_pos_emb_cos.to(hidden_states.device)
|
||||
rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device)
|
||||
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
|
||||
|
||||
grid_thw_np = grid_thw_tensor.cpu().numpy().astype(np.int32)
|
||||
cu_seqlens_np = np.repeat(grid_thw_np[:, 1] * grid_thw_np[:, 2], grid_thw_np[:, 0]).cumsum(
|
||||
axis=0, dtype=np.int32
|
||||
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(self.attn_backend, cu_seqlens, self.device)
|
||||
max_seqlen = torch.tensor(
|
||||
MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
cu_seqlens_np = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens_np])
|
||||
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
self.attn_backend,
|
||||
cu_seqlens_np,
|
||||
cu_seqlens,
|
||||
self.hidden_size,
|
||||
self.tp_size,
|
||||
self.device,
|
||||
)
|
||||
|
||||
hidden_states = hidden_states.unsqueeze(1)
|
||||
|
||||
hidden_states_list = []
|
||||
deepstack_visual_indexes = self.deepstack_visual_indexes
|
||||
|
||||
for layer_num, blk in enumerate(self.blocks):
|
||||
hidden_states = blk(
|
||||
hidden_states,
|
||||
hidden_states = hidden_states + blk.attn(
|
||||
blk.norm1(hidden_states),
|
||||
cu_seqlens=cu_seqlens,
|
||||
rotary_pos_emb_cos=rotary_pos_emb_cos,
|
||||
rotary_pos_emb_sin=rotary_pos_emb_sin,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
hidden_states = hidden_states + blk.mlp(blk.norm2(hidden_states))
|
||||
|
||||
if deepstack_visual_indexes is not None and layer_num in deepstack_visual_indexes:
|
||||
hidden_states_list.append(hidden_states)
|
||||
@@ -784,13 +759,19 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
mm_item_counts,
|
||||
)
|
||||
else:
|
||||
if use_audio_in_video and "audio" in mm_prompt_updates:
|
||||
if use_audio_in_video:
|
||||
# When use_audio_in_video=True, audio is extracted from video and embedded
|
||||
# in the video placeholder tokens. We should:
|
||||
# 1. Filter out audio from prompt updates (audio has no separate placeholder)
|
||||
# 2. Apply remaining updates (video, image, etc.)
|
||||
# 3. Derive audio placeholders from video placeholders
|
||||
filtered_updates = {k: v for k, v in mm_prompt_updates.items() if k != "audio"}
|
||||
prompt_ids, mm_placeholders = self._apply_prompt_updates(
|
||||
prompt_ids,
|
||||
filtered_updates,
|
||||
)
|
||||
mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates)
|
||||
# Derive audio placeholders from video placeholders
|
||||
mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_item_counts)
|
||||
else:
|
||||
prompt_ids, mm_placeholders = self._apply_prompt_updates(
|
||||
prompt_ids,
|
||||
@@ -909,7 +890,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
|
||||
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
|
||||
nonlocal audio_in_video_item_idx
|
||||
audio_num_features = audio_output_lengths[audio_in_video_item_idx]
|
||||
audio_num_features = audio_output_lengths[audio_in_video_item_idx + item_idx]
|
||||
video_grid_thw = out_mm_data["video_grid_thw"][item_idx]
|
||||
|
||||
audio_in_video_item_idx += 1
|
||||
@@ -955,17 +936,27 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
def _derive_audio_from_video_placeholders(
|
||||
self,
|
||||
placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_item_counts: Mapping[str, int],
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
"""
|
||||
Helper to derive audio placeholders from video placeholders when
|
||||
use_audio_in_video=True.
|
||||
|
||||
In use_audio_in_video mode, audio is extracted from video and embedded
|
||||
within the video placeholder tokens. This function creates audio placeholder
|
||||
info by extracting the audio token positions from video placeholders.
|
||||
|
||||
Args:
|
||||
placeholders: Current placeholders (should contain "video")
|
||||
mm_item_counts: Counts of multimodal items from mm_items.get_all_counts()
|
||||
"""
|
||||
if "video" not in placeholders:
|
||||
return placeholders
|
||||
|
||||
# Validate audio and video counts match
|
||||
# In use_audio_in_video mode, audio count comes from mm_items (extracted from video)
|
||||
num_videos = len(placeholders["video"])
|
||||
num_audios = len(mm_prompt_updates.get("audio", []))
|
||||
num_audios = mm_item_counts.get("audio", 0)
|
||||
if num_audios != num_videos:
|
||||
raise ValueError(
|
||||
f"use_audio_in_video requires equal number of audio and video items, got {num_audios=}, {num_videos=}"
|
||||
@@ -1285,7 +1276,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
return []
|
||||
|
||||
# The result multimodal_embeddings is tuple of tensors, with each
|
||||
# tensor corresponding to a multimodal data item (image or video).
|
||||
# tensor correspoending to a multimodal data item (image or video).
|
||||
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
|
||||
|
||||
# NOTE: It is important to iterate over the keys in this dictionary
|
||||
@@ -1309,6 +1300,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self._embed_text_input_ids(
|
||||
input_ids,
|
||||
|
||||
@@ -536,6 +536,10 @@ class Qwen3TTSConfig(PretrainedConfig):
|
||||
if any("Code2Wav" in str(a) for a in archs):
|
||||
if hasattr(config, "rope_parameters"):
|
||||
delattr(config, "rope_parameters")
|
||||
# Code2Wav doesn't use position embeddings, but vLLM validates
|
||||
# max_model_len <= max_position_embeddings. Set a large value
|
||||
# to allow the pipeline.yaml max_model_len (65536) to pass validation.
|
||||
config.max_position_embeddings = 131072
|
||||
return config
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user