diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py index ea9d9dcdd..0b5d78560 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py @@ -91,6 +91,7 @@ class Qwen2_5OmniForConditionalGeneration( self.token2wav = None elif self.model_stage == "talker": + multimodal_config.skip_mm_profiling = True # register the process function for the talker stage self.has_preprocess = True self.set_custom_preprocess(self.talker_preprocess) @@ -103,7 +104,6 @@ class Qwen2_5OmniForConditionalGeneration( # Use registry architecture key architectures=["Qwen2_5OmniTalkerModel"], ) - self.talker.init_multi_modal(thinker_config) self.model = self.talker self.token2wav = None # set suppress start id according to token2wav @@ -124,6 +124,7 @@ class Qwen2_5OmniForConditionalGeneration( self._init_special_tokens_embeddings() elif self.model_stage == "code2wav": + multimodal_config.skip_mm_profiling = True self.thinker = None self.talker = None # Initialize token2wav (code->mel->wav) like thinker/talker @@ -197,6 +198,8 @@ class Qwen2_5OmniForConditionalGeneration( ) -> torch.Tensor: if self.model_stage == "code2wav": return torch.zeros_like(input_ids).reshape(-1, 1).repeat(1, self.vllm_config.model_config.get_hidden_size()) + if self.model_stage == "talker": + return self.model.embed_input_ids(input_ids) return self.model.embed_input_ids( input_ids=input_ids, multimodal_embeddings=multimodal_embeddings, is_multimodal=is_multimodal ) diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py index dbb6c49ef..35d988895 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py @@ -4,42 +4,24 @@ from functools import cached_property import torch import torch.nn as nn from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import Qwen2_5OmniTalkerConfig -from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import Qwen2_5OmniAudioEncoder - -# from vllm.attention import AttentionMetadata # unused import from vllm.config import VllmConfig from vllm.logger import init_logger -from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP -from vllm.model_executor.models.qwen2_5_omni_thinker import ( - Qwen2_5OmniThinkerProcessingInfo, -) -from vllm.model_executor.models.qwen2_5_vl import Qwen2_5_VisionTransformer +from vllm.model_executor.models.interfaces import SupportsPP from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, init_vllm_registered_model, maybe_prefix, ) -from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.sample.sampler import Sampler -from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( - Qwen2_5OmniConditionalGenerationMixin, - Qwen2_5OmniThinkerDummyInputsBuilder, - Qwen2_5OmniThinkerMultiModalProcessor, -) - -@MULTIMODAL_REGISTRY.register_processor( - Qwen2_5OmniThinkerMultiModalProcessor, - info=Qwen2_5OmniThinkerProcessingInfo, - dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder, -) class Qwen2_5OmniTalkerForConditionalGeneration( - nn.Module, SupportsMultiModal, SupportsPP, Qwen2_5OmniConditionalGenerationMixin + nn.Module, + SupportsPP, ): logger = init_logger(__name__) # Align to thinker-style static mapper for clarity @@ -84,15 +66,6 @@ class Qwen2_5OmniTalkerForConditionalGeneration( # suppress start id self.suppress_start_id = None - def init_multi_modal(self, thinker_config): - self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config) - self.visual = Qwen2_5_VisionTransformer( - vision_config=thinker_config.vision_config, - norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6), - quant_config=self.quant_config, - prefix=maybe_prefix(self.prefix, "visual"), - ) - def get_language_model(self) -> torch.nn.Module: return self.language_model @@ -103,22 +76,9 @@ class Qwen2_5OmniTalkerForConditionalGeneration( return Sampler() - def embed_input_ids( - self, - input_ids: torch.Tensor, - multimodal_embeddings: MultiModalEmbeddings | None = None, - *, - is_multimodal: torch.Tensor | None = None, - ) -> torch.Tensor: - # This is to satisfy the type checker for each overload - if multimodal_embeddings is None or is_multimodal is None: - return super().embed_input_ids(input_ids) - - return super().embed_input_ids( - input_ids, - multimodal_embeddings=multimodal_embeddings, - is_multimodal=is_multimodal, - ) + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + """Embed codec input IDs.""" + return self.language_model.embed_input_ids(input_ids) def forward( self, @@ -129,12 +89,10 @@ class Qwen2_5OmniTalkerForConditionalGeneration( **kwargs: object, ) -> torch.Tensor | IntermediateTensors: assert input_ids is not None or inputs_embeds is not None, "input_ids or inputs_embeds must be provided" - # forward_context: ForwardContext = get_forward_context() # unused variable if intermediate_tensors is not None: inputs_embeds = None elif inputs_embeds is None: - # for profile_run: inputs_embeds = self.embed_input_ids(input_ids) input_ids = None @@ -202,52 +160,8 @@ class Qwen2_5OmniTalkerForConditionalGeneration( ) except Exception: pass - multi_model_weights = set() - for name, param in self.visual.named_parameters(): - multi_model_weights.add("visual." + name) - for name, param in self.audio_tower.named_parameters(): - multi_model_weights.add("audio_tower." + name) - loaded.update(multi_model_weights) return loaded - def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} - - # Preserve the order of modalities if there are multiple of them - # from the order of kwargs. - for input_key in kwargs: - if input_key in ("pixel_values", "image_embeds") and "image" not in mm_input_by_modality: - mm_input_by_modality["image"] = self._parse_and_validate_image_input(**kwargs) - if input_key in ("pixel_values_videos", "video_embeds") and "video" not in mm_input_by_modality: - mm_input_by_modality["video"] = self._parse_and_validate_video_input(**kwargs) - if input_key in ("input_audio_features") and "audio" not in mm_input_by_modality: - mm_input_by_modality["audio"] = self._parse_and_validate_audio_input(**kwargs) - return mm_input_by_modality - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) - if not mm_input_by_modality: - return [] - - # The result multimodal_embeddings is tuple of tensors, with each - # tensor corresponding to a multimodal data item (image or video). - multimodal_embeddings: tuple[torch.Tensor, ...] = () - - # NOTE: It is important to iterate over the keys in this dictionary - # to preserve the order of the modalities. - for modality in mm_input_by_modality: - multimodal_input = mm_input_by_modality[modality] - if modality == "image": - vision_embeddings = self._process_image_input(multimodal_input) - multimodal_embeddings += vision_embeddings - if modality == "video": - video_embeddings = self._process_video_input(multimodal_input) - multimodal_embeddings += video_embeddings - if modality == "audio": - audio_embeddings = self._process_audio_input(multimodal_input) - multimodal_embeddings += audio_embeddings - return multimodal_embeddings - def set_suppress_start_id(self, start_id: int): self.suppress_start_id = start_id self.logger.debug(f"Set suppress start id to {self.suppress_start_id}")