[Bugfix] Fix Compatibility of Ming-flash-omni-2.0 on transformers 5.X and vllm 0.22 (#4080)
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com> Signed-off-by: akshatvishu <akshatnayak197@gmail.com> Co-authored-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
1 parent
6e8ef213d8
commit
95d56cf062
6 files changed
+97
-26
No files matched your search
@@ -72,6 +72,7 @@ from vllm_omni.transformers_utils.processors.ming import (
|
||||
PLACEHOLDER_VIDEO_TOKEN_IN_TEXT,
|
||||
MingFlashOmniProcessor,
|
||||
MingWhisperFeatureExtractor,
|
||||
raise_missing_video_processor,
|
||||
)
|
||||
|
||||
from .audio_encoder import WhisperAudioEncoder
|
||||
@@ -540,20 +541,20 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
|
||||
if images is not None:
|
||||
image_outputs = hf_processor.image_processor(
|
||||
images=images,
|
||||
videos=None,
|
||||
return_tensors="pt",
|
||||
)
|
||||
data.update(image_outputs)
|
||||
|
||||
videos = mm_data.get("videos", None)
|
||||
if videos is not None:
|
||||
# TODO: ``videos=`` on image_processor is deprecated since
|
||||
# transformers v4.57 (removed in v5); migrate to Qwen2VLVideoProcessor.
|
||||
video_outputs = hf_processor.image_processor(
|
||||
images=None,
|
||||
videos=videos,
|
||||
return_tensors="pt",
|
||||
)
|
||||
video_processor = getattr(hf_processor, "video_processor", None)
|
||||
if video_processor is not None:
|
||||
video_outputs = video_processor(
|
||||
videos=videos,
|
||||
return_tensors="pt",
|
||||
)
|
||||
else:
|
||||
raise_missing_video_processor()
|
||||
# Rename keys to distinguish from images
|
||||
if "pixel_values" in video_outputs:
|
||||
video_outputs["pixel_values_videos"] = video_outputs.pop("pixel_values")
|
||||
|
||||
@@ -714,6 +714,9 @@ class BailingMoeV2Model(nn.Module):
|
||||
def get_input_embeddings(self):
|
||||
return self.word_embeddings
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.word_embeddings(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -794,6 +797,9 @@ class BailingMoeV2ForCausalLM(nn.Module, CustomProcessMixin):
|
||||
self.sampler = Sampler()
|
||||
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -818,7 +824,7 @@ class BailingMoeV2ForCausalLM(nn.Module, CustomProcessMixin):
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, sampling_metadata)
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def sample(
|
||||
|
||||
@@ -902,6 +902,7 @@ class MingAudioGenerator:
|
||||
if self._audio_vae is None:
|
||||
return requested_max_steps
|
||||
|
||||
# Transformers >=5.x may expose these config values as 0-d tensors.
|
||||
sample_rate = float(self._audio_vae.config.sample_rate)
|
||||
vae_patch_size = float(getattr(self._audio_vae.config, "patch_size", 4))
|
||||
hop_size = float(getattr(self._audio_vae.decoder, "hop_length", 320))
|
||||
@@ -1041,7 +1042,7 @@ class MingAudioGenerator:
|
||||
use_cache=True,
|
||||
)
|
||||
else:
|
||||
past_seen_tokens = past_key_values.get_seq_length()
|
||||
past_seen_tokens = int(past_key_values.get_seq_length())
|
||||
cache_position = torch.arange(
|
||||
past_seen_tokens,
|
||||
past_seen_tokens + inputs_embeds.shape[1],
|
||||
|
||||
Reference in new issue
Block a user