rebase: align vllm-omni with vLLM e9163b536e72
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
f137a1e3ad
commit
3a508d7b7a
14 files changed
+97
-46
No files matched your search
@@ -773,7 +773,6 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# This is to satisfy the type checker for each overload
|
||||
if multimodal_embeddings is None or is_multimodal is None:
|
||||
@@ -783,7 +782,6 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
|
||||
input_ids,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
def base_local_forward(
|
||||
|
||||
@@ -109,7 +109,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# This is to satisfy the type checker for each overload
|
||||
if multimodal_embeddings is None or is_multimodal is None:
|
||||
@@ -119,7 +118,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
|
||||
input_ids,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -116,6 +116,15 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
|
||||
use_audio_in_video = audio_pad_id not in prompt_ids
|
||||
# for mutilmodality cache
|
||||
if any(item is None for item in mm_kwargs["video"]):
|
||||
video_token_id = self.info.get_hf_config().video_token_id
|
||||
audio_token_id = self.info.get_hf_config().audio_token_id
|
||||
video_audio_item_num = sum(id in (video_token_id, audio_token_id) for id in prompt_ids)
|
||||
audio_updates_num = len(mm_prompt_updates.get("audio", []))
|
||||
video_updates_num = len(mm_prompt_updates.get("video", []))
|
||||
if video_audio_item_num != video_updates_num + audio_updates_num:
|
||||
use_audio_in_video = True
|
||||
|
||||
if is_update_applied:
|
||||
mm_placeholders = self._find_mm_placeholders(
|
||||
@@ -600,7 +609,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if multimodal_embeddings is None or is_multimodal is None:
|
||||
return super().embed_input_ids(input_ids)
|
||||
@@ -609,7 +617,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
input_ids,
|
||||
self.get_language_model().embed_input_ids,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
if len(multimodal_embeddings) == 0:
|
||||
@@ -632,7 +639,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
input_ids,
|
||||
self.get_language_model().embed_input_ids,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
return merge_interleaved_embeddings(
|
||||
inputs_embeds,
|
||||
@@ -649,7 +655,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
input_ids,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -146,9 +146,11 @@ class Qwen3Omni_VisionTransformer(_Qwen3Omni_VisionTransformer):
|
||||
cu_seqlens = np.repeat(grid_thw_np[:, 1] * grid_thw_np[:, 2], grid_thw_np[:, 0]).cumsum(axis=0, dtype=np.int32)
|
||||
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
|
||||
|
||||
sequence_lengths = MMEncoderAttention.maybe_compute_sequence_lengths(self.attn_backend, cu_seqlens)
|
||||
if sequence_lengths is not None:
|
||||
sequence_lengths = torch.from_numpy(sequence_lengths).to(self.device, non_blocking=True)
|
||||
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
self.attn_backend,
|
||||
cu_seqlens,
|
||||
self.device,
|
||||
)
|
||||
max_seqlen = torch.tensor(
|
||||
MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens),
|
||||
dtype=torch.int32,
|
||||
@@ -475,6 +477,15 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
|
||||
use_audio_in_video = audio_pad_id not in prompt_ids
|
||||
# for mutilmodality cache
|
||||
if any(item is None for item in mm_kwargs["video"]):
|
||||
video_token_id = self.info.get_hf_config().video_token_id
|
||||
audio_token_id = self.info.get_hf_config().audio_token_id
|
||||
video_audio_item_num = sum(id in (video_token_id, audio_token_id) for id in prompt_ids)
|
||||
audio_updates_num = len(mm_prompt_updates.get("audio", []))
|
||||
video_updates_num = len(mm_prompt_updates.get("video", []))
|
||||
if video_audio_item_num != video_updates_num + audio_updates_num:
|
||||
use_audio_in_video = True
|
||||
|
||||
# normal case with `use_audio_in_video=False`
|
||||
if is_update_applied:
|
||||
@@ -966,13 +977,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self._embed_text_input_ids(
|
||||
input_ids,
|
||||
self.language_model.embed_input_ids,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
@@ -1064,7 +1073,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
input_ids,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in new issue
Block a user