anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Perf] Remove dead audio_tower and visual from Qwen2.5-Omni talker stage (#3425)

Browse Source 
Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nick CaoandClaude authored and GitHub committed 2026-05-12 23:24:13 +08:00
1 parent f4b28f2398
commit dca369d448
2 files changed
+10 -93
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py
+4 -1
View File
@@ -91,6 +91,7 @@ class Qwen2_5OmniForConditionalGeneration(
self.token2wav = None
elif self.model_stage == "talker":
multimodal_config.skip_mm_profiling = True
# register the process function for the talker stage
self.has_preprocess = True
self.set_custom_preprocess(self.talker_preprocess)
@@ -103,7 +104,6 @@ class Qwen2_5OmniForConditionalGeneration(
# Use registry architecture key
architectures=["Qwen2_5OmniTalkerModel"],
)
self.talker.init_multi_modal(thinker_config)
self.model = self.talker
self.token2wav = None
# set suppress start id according to token2wav
@@ -124,6 +124,7 @@ class Qwen2_5OmniForConditionalGeneration(
self._init_special_tokens_embeddings()
elif self.model_stage == "code2wav":
multimodal_config.skip_mm_profiling = True
self.thinker = None
self.talker = None
# Initialize token2wav (code->mel->wav) like thinker/talker
@@ -197,6 +198,8 @@ class Qwen2_5OmniForConditionalGeneration(
) -> torch.Tensor:
if self.model_stage == "code2wav":
return torch.zeros_like(input_ids).reshape(-1, 1).repeat(1, self.vllm_config.model_config.get_hidden_size())
if self.model_stage == "talker":
return self.model.embed_input_ids(input_ids)
return self.model.embed_input_ids(
input_ids=input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal
)
vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py
+6 -92
View File
@@ -4,42 +4,24 @@ from functools import cached_property
import torch
import torch.nn as nn
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniTalkerConfig
from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder
# from vllm.attention import AttentionMetadata # unused import
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from vllm.model_executor.models.qwen2_5_omni_thinker import (
Qwen2_5OmniThinkerProcessingInfo,
)
from vllm.model_executor.models.qwen2_5_vl import Qwen2_5_VisionTransformer
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
init_vllm_registered_model,
maybe_prefix,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
Qwen2_5OmniConditionalGenerationMixin,
Qwen2_5OmniThinkerDummyInputsBuilder,
Qwen2_5OmniThinkerMultiModalProcessor,
)
@MULTIMODAL_REGISTRY.register_processor(
Qwen2_5OmniThinkerMultiModalProcessor,
info=Qwen2_5OmniThinkerProcessingInfo,
dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder,
)
class Qwen2_5OmniTalkerForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, Qwen2_5OmniConditionalGenerationMixin
nn.Module,
SupportsPP,
):
logger = init_logger(__name__)
# Align to thinker-style static mapper for clarity
@@ -84,15 +66,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
# suppress start id
self.suppress_start_id = None
def init_multi_modal(self, thinker_config):
self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
self.visual = Qwen2_5_VisionTransformer(
vision_config=thinker_config.vision_config,
norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
quant_config=self.quant_config,
prefix=maybe_prefix(self.prefix, "visual"),
)
def get_language_model(self) -> torch.nn.Module:
return self.language_model
@@ -103,22 +76,9 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
return Sampler()
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
# This is to satisfy the type checker for each overload
if multimodal_embeddings is None or is_multimodal is None:
return super().embed_input_ids(input_ids)
return super().embed_input_ids(
input_ids,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
"""Embed codec input IDs."""
return self.language_model.embed_input_ids(input_ids)
def forward(
self,
@@ -129,12 +89,10 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
assert input_ids is not None or inputs_embeds is not None, "input_ids or inputs_embeds must be provided"
# forward_context: ForwardContext = get_forward_context() # unused variable
if intermediate_tensors is not None:
inputs_embeds = None
elif inputs_embeds is None:
# for profile_run:
inputs_embeds = self.embed_input_ids(input_ids)
input_ids = None
@@ -202,52 +160,8 @@ class Qwen2_5OmniTalkerForConditionalGeneration(
)
except Exception:
pass
multi_model_weights = set()
for name, param in self.visual.named_parameters():
multi_model_weights.add("visual." + name)
for name, param in self.audio_tower.named_parameters():
multi_model_weights.add("audio_tower." + name)
loaded.update(multi_model_weights)
return loaded
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
for input_key in kwargs:
if input_key in ("pixel_values", "image_embeds") and "image" not in mm_input_by_modality:
mm_input_by_modality["image"] = self._parse_and_validate_image_input(**kwargs)
if input_key in ("pixel_values_videos", "video_embeds") and "video" not in mm_input_by_modality:
mm_input_by_modality["video"] = self._parse_and_validate_video_input(**kwargs)
if input_key in ("input_audio_features") and "audio" not in mm_input_by_modality:
mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(**kwargs)
return mm_input_by_modality
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
if not mm_input_by_modality:
return []
# The result multimodal_embeddings is tuple of tensors, with each
# tensor corresponding to a multimodal data item (image or video).
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
# NOTE: It is important to iterate over the keys in this dictionary
# to preserve the order of the modalities.
for modality in mm_input_by_modality:
multimodal_input = mm_input_by_modality[modality]
if modality == "image":
vision_embeddings = self._process_image_input(multimodal_input)
multimodal_embeddings += vision_embeddings
if modality == "video":
video_embeddings = self._process_video_input(multimodal_input)
multimodal_embeddings += video_embeddings
if modality == "audio":
audio_embeddings = self._process_audio_input(multimodal_input)
multimodal_embeddings += audio_embeddings
return multimodal_embeddings
def set_suppress_start_id(self, start_id: int):
self.suppress_start_id = start_id
self.logger.debug(f"Set suppress start id to {self.suppress_start_id}")