[Bugfix] Fix Compatibility of Ming-flash-omni-2.0 on transformers 5.X and vllm 0.22 (#4080)

Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
Co-authored-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
Yuanheng Zhaoandakshatvishu authored and GitHub committed 2026-06-05 09:27:42 +08:00
1 parent 6e8ef213d8
commit 95d56cf062
6 files changed
+97 -26

No files matched your search

@@ -72,6 +72,7 @@ from vllm_omni.transformers_utils.processors.ming import (
PLACEHOLDER_VIDEO_TOKEN_IN_TEXT,
MingFlashOmniProcessor,
MingWhisperFeatureExtractor,
raise_missing_video_processor,
)
from .audio_encoder import WhisperAudioEncoder
@@ -540,20 +541,20 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
if images is not None:
image_outputs = hf_processor.image_processor(
images=images,
videos=None,
return_tensors="pt",
)
data.update(image_outputs)
videos = mm_data.get("videos", None)
if videos is not None:
# TODO: ``videos=`` on image_processor is deprecated since
# transformers v4.57 (removed in v5); migrate to Qwen2VLVideoProcessor.
video_outputs = hf_processor.image_processor(
images=None,
videos=videos,
return_tensors="pt",
)
video_processor = getattr(hf_processor, "video_processor", None)
if video_processor is not None:
video_outputs = video_processor(
videos=videos,
return_tensors="pt",
)
else:
raise_missing_video_processor()
# Rename keys to distinguish from images
if "pixel_values" in video_outputs:
video_outputs["pixel_values_videos"] = video_outputs.pop("pixel_values")
@@ -714,6 +714,9 @@ class BailingMoeV2Model(nn.Module):
def get_input_embeddings(self):
return self.word_embeddings
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.word_embeddings(input_ids)
def forward(
self,
input_ids: torch.Tensor,
@@ -794,6 +797,9 @@ class BailingMoeV2ForCausalLM(nn.Module, CustomProcessMixin):
self.sampler = Sampler()
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor,
@@ -818,7 +824,7 @@ class BailingMoeV2ForCausalLM(nn.Module, CustomProcessMixin):
hidden_states: torch.Tensor,
sampling_metadata,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states, sampling_metadata)
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def sample(
@@ -902,6 +902,7 @@ class MingAudioGenerator:
if self._audio_vae is None:
return requested_max_steps
# Transformers >=5.x may expose these config values as 0-d tensors.
sample_rate = float(self._audio_vae.config.sample_rate)
vae_patch_size = float(getattr(self._audio_vae.config, "patch_size", 4))
hop_size = float(getattr(self._audio_vae.decoder, "hop_length", 320))
@@ -1041,7 +1042,7 @@ class MingAudioGenerator:
use_cache=True,
)
else:
past_seen_tokens = past_key_values.get_seq_length()
past_seen_tokens = int(past_key_values.get_seq_length())
cache_position = torch.arange(
past_seen_tokens,
past_seen_tokens + inputs_embeds.shape[1],