diff --git a/examples/offline_inference/qwen3_omni/end2end.py b/examples/offline_inference/qwen3_omni/end2end.py index 4d1a49fb5..4d3ce25cd 100644 --- a/examples/offline_inference/qwen3_omni/end2end.py +++ b/examples/offline_inference/qwen3_omni/end2end.py @@ -143,11 +143,61 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin ) +def get_mixed_modalities_query() -> QueryResult: + question = "What is recited in the audio? What is the content of this image? Why is this video funny?" + prompt = ( + f"<|im_start|>system\n{default_system}<|im_end|>\n" + "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>" + "<|vision_start|><|image_pad|><|vision_end|>" + "<|vision_start|><|video_pad|><|vision_end|>" + f"{question}<|im_end|>\n" + f"<|im_start|>assistant\n" + ) + return QueryResult( + inputs={ + "prompt": prompt, + "multi_modal_data": { + "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate, + "image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"), + "video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays, + }, + }, + limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1}, + ) + + +def get_multi_audios_query() -> QueryResult: + question = "Are these two audio clips the same?" + prompt = ( + f"<|im_start|>system\n{default_system}<|im_end|>\n" + "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>" + "<|audio_start|><|audio_pad|><|audio_end|>" + f"{question}<|im_end|>\n" + f"<|im_start|>assistant\n" + ) + return QueryResult( + inputs={ + "prompt": prompt, + "multi_modal_data": { + "audio": [ + AudioAsset("winning_call").audio_and_sample_rate, + AudioAsset("mary_had_lamb").audio_and_sample_rate, + ], + }, + }, + limit_mm_per_prompt={ + "audio": 2, + }, + ) + + query_map = { "text": get_text_query, "use_audio": get_audio_query, "use_image": get_image_query, "use_video": get_video_query, + "use_mixed_modalities": get_mixed_modalities_query, + "use_multi_audios": get_multi_audios_query, } @@ -276,7 +326,7 @@ def parse_args(): "--query-type", "-q", type=str, - default="use_video", + default="use_mixed_modalities", choices=query_map.keys(), help="Query type.", ) diff --git a/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py index 7ebf8c7f4..ca361ef15 100644 --- a/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py +++ b/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py @@ -254,11 +254,64 @@ def get_audio_query(audio_path: str | None = None, custom_prompt: str | None = N return prompt +def get_mixed_modalities_query( + video_path: str | None = None, + image_path: str | None = None, + audio_path: str | None = None, + custom_prompt: str | None = None, +): + """ + Online-friendly multimodal user message: + - Uses URLs (or base64 data URLs) for audio / image / video. + - Returns the OpenAI-style message dict directly (not the offline QueryResult). + """ + question = ( + custom_prompt or "What is recited in the audio? What is the content of this image? Why is this video funny?" + ) + + audio_url = get_audio_url_from_path(audio_path) + image_url = get_image_url_from_path(image_path) + video_url = get_video_url_from_path(video_path) + + return { + "role": "user", + "content": [ + {"type": "audio_url", "audio_url": {"url": audio_url}}, + {"type": "image_url", "image_url": {"url": image_url}}, + {"type": "video_url", "video_url": {"url": video_url}}, + {"type": "text", "text": question}, + ], + } + + +def get_multi_audios_query(custom_prompt: str | None = None): + """ + Online-friendly two-audio comparison request. + - Encodes both audio clips as URLs (or data URLs). + - Returns the OpenAI-style message dict. + """ + question = custom_prompt or "Are these two audio clips the same?" + # Use default demo clips; you can point to your own via --audio-path if needed. + audio_url_1 = get_audio_url_from_path(AudioAsset("winning_call").url) + audio_url_2 = get_audio_url_from_path(AudioAsset("mary_had_lamb").url) + + return { + "role": "user", + "content": [ + {"type": "audio_url", "audio_url": {"url": audio_url_1}}, + {"type": "audio_url", "audio_url": {"url": audio_url_2}}, + {"type": "text", "text": question}, + ], + } + + query_map = { "text": get_text_query, "use_audio": get_audio_query, "use_image": get_image_query, "use_video": get_video_query, + "use_mixed_modalities": get_mixed_modalities_query, + "use_multi_audios": get_multi_audios_query, } @@ -363,7 +416,7 @@ def parse_args(): "--query-type", "-q", type=str, - default="use_video", + default="use_mixed_modalities", choices=query_map.keys(), help="Query type.", ) diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py index e8536a500..729d9c318 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py @@ -25,6 +25,7 @@ from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_p # from vllm.model_executor.models.qwen2_code2wav_dit import Qwen2Code2wav from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata @@ -338,6 +339,8 @@ class Qwen2_5OmniForConditionalGeneration( def get_mrope_input_positions( self, input_tokens: list[int], + mm_features: list[MultiModalFeatureSpec] | None = None, + *, hf_config: PretrainedConfig, image_grid_thw: list[list[int]] | torch.Tensor, video_grid_thw: list[list[int]] | torch.Tensor, diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py index 2dbc71c96..7695adec7 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py @@ -16,7 +16,8 @@ from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import ( ) from vllm.config import VllmConfig from vllm.logger import init_logger -from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding +from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP from vllm.model_executor.models.qwen3_omni_moe_thinker import ( Qwen3OmniMoeConditionalGenerationMixin, Qwen3OmniMoeThinkerDummyInputsBuilder, @@ -25,6 +26,7 @@ from vllm.model_executor.models.qwen3_omni_moe_thinker import ( ) from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata @@ -66,7 +68,7 @@ logger = init_logger(__name__) dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder, ) class Qwen3OmniMoeForConditionalGeneration( - nn.Module, SupportsMultiModal, SupportsPP, Qwen3OmniMoeConditionalGenerationMixin, CustomProcessMixin + nn.Module, SupportsMultiModal, SupportsPP, Qwen3OmniMoeConditionalGenerationMixin, CustomProcessMixin, SupportsMRoPE ): """ Unified Qwen3 Omni MoE model combining thinker, talker, and code2wav. @@ -259,6 +261,19 @@ class Qwen3OmniMoeForConditionalGeneration( if i != self.config.talker_config.codec_eos_token_id ] + def get_mrope_input_positions( + self, + input_tokens: list[int], + mm_features: list[MultiModalFeatureSpec] | None = None, + **kwargs: object, + ) -> tuple[torch.Tensor, int]: + if self.model_stage == "thinker": + if mm_features is None: + msg = "Qwen3 Omni thinker get_mrope_input_positions requires mm_features" + raise ValueError(msg) + return self.thinker.get_mrope_input_positions(input_tokens, mm_features) + return MRotaryEmbedding.get_input_positions_tensor(input_tokens, **kwargs) + def forward( self, input_ids: torch.Tensor, @@ -330,22 +345,17 @@ class Qwen3OmniMoeForConditionalGeneration( if is_npu(): # TODO: remove this hack when NPU supports batched inputs properly thinker_input_ids = input_ids[0] if input_ids is not None and _added_batch_dim else input_ids - thinker_positions = positions[0] if positions.ndim > 1 else positions thinker_inputs_embeds = ( inputs_embeds[0] if inputs_embeds is not None and _added_batch_dim else inputs_embeds ) else: thinker_input_ids = input_ids - thinker_positions = positions[0] if positions.ndim > 1 else positions thinker_inputs_embeds = inputs_embeds - # thinker_input_ids = input_ids - # thinker_positions = positions[0] if positions.ndim > 1 else positions - # thinker_inputs_embeds = inputs_embeds # Run thinker text_hidden_states, captured_layer_dict = self.thinker( input_ids=thinker_input_ids, - positions=thinker_positions, + positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=thinker_inputs_embeds, **capture_kwargs, diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index e426e3938..2cb76815a 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -77,6 +77,7 @@ class OmniGPUModelRunner(GPUModelRunner): if supports_mrope(self.model): req_state.mrope_positions, req_state.mrope_position_delta = self.model.get_mrope_input_positions( req_state.prompt_token_ids, + mm_features=req_state.mm_features, hf_config=self.model_config.hf_config, image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw,