rebase: align vllm-omni with vLLM e9163b536e72

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-03-16 02:32:39 +00:00
1 parent f137a1e3ad
commit 3a508d7b7a
14 files changed
+97 -46

No files matched your search

@@ -773,7 +773,6 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
# This is to satisfy the type checker for each overload
if multimodal_embeddings is None or is_multimodal is None:
@@ -783,7 +782,6 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
input_ids,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
def base_local_forward(
@@ -109,7 +109,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
# This is to satisfy the type checker for each overload
if multimodal_embeddings is None or is_multimodal is None:
@@ -119,7 +118,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
input_ids,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
def forward(
@@ -116,6 +116,15 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
tokenizer = self.info.get_tokenizer()
audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
use_audio_in_video = audio_pad_id not in prompt_ids
# for mutilmodality cache
if any(item is None for item in mm_kwargs["video"]):
video_token_id = self.info.get_hf_config().video_token_id
audio_token_id = self.info.get_hf_config().audio_token_id
video_audio_item_num = sum(id in (video_token_id, audio_token_id) for id in prompt_ids)
audio_updates_num = len(mm_prompt_updates.get("audio", []))
video_updates_num = len(mm_prompt_updates.get("video", []))
if video_audio_item_num != video_updates_num + audio_updates_num:
use_audio_in_video = True
if is_update_applied:
mm_placeholders = self._find_mm_placeholders(
@@ -600,7 +609,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
if multimodal_embeddings is None or is_multimodal is None:
return super().embed_input_ids(input_ids)
@@ -609,7 +617,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
input_ids,
self.get_language_model().embed_input_ids,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
if len(multimodal_embeddings) == 0:
@@ -632,7 +639,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
input_ids,
self.get_language_model().embed_input_ids,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
return merge_interleaved_embeddings(
inputs_embeds,
@@ -649,7 +655,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
input_ids,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
def forward(
@@ -146,9 +146,11 @@ class Qwen3Omni_VisionTransformer(_Qwen3Omni_VisionTransformer):
cu_seqlens = np.repeat(grid_thw_np[:, 1] * grid_thw_np[:, 2], grid_thw_np[:, 0]).cumsum(axis=0, dtype=np.int32)
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
sequence_lengths = MMEncoderAttention.maybe_compute_sequence_lengths(self.attn_backend, cu_seqlens)
if sequence_lengths is not None:
sequence_lengths = torch.from_numpy(sequence_lengths).to(self.device, non_blocking=True)
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
self.attn_backend,
cu_seqlens,
self.device,
)
max_seqlen = torch.tensor(
MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens),
dtype=torch.int32,
@@ -475,6 +477,15 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
tokenizer = self.info.get_tokenizer()
audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
use_audio_in_video = audio_pad_id not in prompt_ids
# for mutilmodality cache
if any(item is None for item in mm_kwargs["video"]):
video_token_id = self.info.get_hf_config().video_token_id
audio_token_id = self.info.get_hf_config().audio_token_id
video_audio_item_num = sum(id in (video_token_id, audio_token_id) for id in prompt_ids)
audio_updates_num = len(mm_prompt_updates.get("audio", []))
video_updates_num = len(mm_prompt_updates.get("video", []))
if video_audio_item_num != video_updates_num + audio_updates_num:
use_audio_in_video = True
# normal case with `use_audio_in_video=False`
if is_update_applied:
@@ -966,13 +977,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
inputs_embeds = self._embed_text_input_ids(
input_ids,
self.language_model.embed_input_ids,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
@@ -1064,7 +1073,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
input_ids,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
handle_oov_mm_token=handle_oov_mm_token,
)
def forward(