[Debug] Debug qwen3 mix modality output empty string (#431)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>

examples/offline_inference/qwen3_omni/end2end.py

@@ -143,11 +143,61 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin
     )
 
 
+def get_mixed_modalities_query() -> QueryResult:
+    question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
+    prompt = (
+        f"<|im_start|>system\n{default_system}<|im_end|>\n"
+        "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
+        "<|vision_start|><|image_pad|><|vision_end|>"
+        "<|vision_start|><|video_pad|><|vision_end|>"
+        f"{question}<|im_end|>\n"
+        f"<|im_start|>assistant\n"
+    )
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {
+                "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
+                "image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
+                "video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
+            },
+        },
+        limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
+    )
+
+
+def get_multi_audios_query() -> QueryResult:
+    question = "Are these two audio clips the same?"
+    prompt = (
+        f"<|im_start|>system\n{default_system}<|im_end|>\n"
+        "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
+        "<|audio_start|><|audio_pad|><|audio_end|>"
+        f"{question}<|im_end|>\n"
+        f"<|im_start|>assistant\n"
+    )
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {
+                "audio": [
+                    AudioAsset("winning_call").audio_and_sample_rate,
+                    AudioAsset("mary_had_lamb").audio_and_sample_rate,
+                ],
+            },
+        },
+        limit_mm_per_prompt={
+            "audio": 2,
+        },
+    )
+
+
 query_map = {
     "text": get_text_query,
     "use_audio": get_audio_query,
     "use_image": get_image_query,
     "use_video": get_video_query,
+    "use_mixed_modalities": get_mixed_modalities_query,
+    "use_multi_audios": get_multi_audios_query,
 }
 
 
@@ -276,7 +326,7 @@ def parse_args():
         "--query-type",
         "-q",
         type=str,
-        default="use_video",
+        default="use_mixed_modalities",
         choices=query_map.keys(),
         help="Query type.",
     )

examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -254,11 +254,64 @@ def get_audio_query(audio_path: str | None = None, custom_prompt: str | None = N
     return prompt
 
 
+def get_mixed_modalities_query(
+    video_path: str | None = None,
+    image_path: str | None = None,
+    audio_path: str | None = None,
+    custom_prompt: str | None = None,
+):
+    """
+    Online-friendly multimodal user message:
+    - Uses URLs (or base64 data URLs) for audio / image / video.
+    - Returns the OpenAI-style message dict directly (not the offline QueryResult).
+    """
+    question = (
+        custom_prompt or "What is recited in the audio? What is the content of this image? Why is this video funny?"
+    )
+
+    audio_url = get_audio_url_from_path(audio_path)
+    image_url = get_image_url_from_path(image_path)
+    video_url = get_video_url_from_path(video_path)
+
+    return {
+        "role": "user",
+        "content": [
+            {"type": "audio_url", "audio_url": {"url": audio_url}},
+            {"type": "image_url", "image_url": {"url": image_url}},
+            {"type": "video_url", "video_url": {"url": video_url}},
+            {"type": "text", "text": question},
+        ],
+    }
+
+
+def get_multi_audios_query(custom_prompt: str | None = None):
+    """
+    Online-friendly two-audio comparison request.
+    - Encodes both audio clips as URLs (or data URLs).
+    - Returns the OpenAI-style message dict.
+    """
+    question = custom_prompt or "Are these two audio clips the same?"
+    # Use default demo clips; you can point to your own via --audio-path if needed.
+    audio_url_1 = get_audio_url_from_path(AudioAsset("winning_call").url)
+    audio_url_2 = get_audio_url_from_path(AudioAsset("mary_had_lamb").url)
+
+    return {
+        "role": "user",
+        "content": [
+            {"type": "audio_url", "audio_url": {"url": audio_url_1}},
+            {"type": "audio_url", "audio_url": {"url": audio_url_2}},
+            {"type": "text", "text": question},
+        ],
+    }
+
+
 query_map = {
     "text": get_text_query,
     "use_audio": get_audio_query,
     "use_image": get_image_query,
     "use_video": get_video_query,
+    "use_mixed_modalities": get_mixed_modalities_query,
+    "use_multi_audios": get_multi_audios_query,
 }
 
 
@@ -363,7 +416,7 @@ def parse_args():
         "--query-type",
         "-q",
         type=str,
-        default="use_video",
+        default="use_mixed_modalities",
         choices=query_map.keys(),
         help="Query type.",
     )

vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py

@@ -25,6 +25,7 @@ from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_p
 
 # from vllm.model_executor.models.qwen2_code2wav_dit import Qwen2Code2wav
 from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import MultiModalFeatureSpec
 from vllm.sequence import IntermediateTensors
 from vllm.v1.outputs import SamplerOutput
 from vllm.v1.sample.metadata import SamplingMetadata
@@ -338,6 +339,8 @@ class Qwen2_5OmniForConditionalGeneration(
     def get_mrope_input_positions(
         self,
         input_tokens: list[int],
+        mm_features: list[MultiModalFeatureSpec] | None = None,
+        *,
         hf_config: PretrainedConfig,
         image_grid_thw: list[list[int]] | torch.Tensor,
         video_grid_thw: list[list[int]] | torch.Tensor,

vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py

@@ -16,7 +16,8 @@ from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import (
 )
 from vllm.config import VllmConfig
 from vllm.logger import init_logger
-from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
+from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
+from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP
 from vllm.model_executor.models.qwen3_omni_moe_thinker import (
     Qwen3OmniMoeConditionalGenerationMixin,
     Qwen3OmniMoeThinkerDummyInputsBuilder,
@@ -25,6 +26,7 @@ from vllm.model_executor.models.qwen3_omni_moe_thinker import (
 )
 from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
 from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import MultiModalFeatureSpec
 from vllm.sequence import IntermediateTensors
 from vllm.v1.outputs import SamplerOutput
 from vllm.v1.sample.metadata import SamplingMetadata
@@ -66,7 +68,7 @@ logger = init_logger(__name__)
     dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
 )
 class Qwen3OmniMoeForConditionalGeneration(
-    nn.Module, SupportsMultiModal, SupportsPP, Qwen3OmniMoeConditionalGenerationMixin, CustomProcessMixin
+    nn.Module, SupportsMultiModal, SupportsPP, Qwen3OmniMoeConditionalGenerationMixin, CustomProcessMixin, SupportsMRoPE
 ):
     """
     Unified Qwen3 Omni MoE model combining thinker, talker, and code2wav.
@@ -259,6 +261,19 @@ class Qwen3OmniMoeForConditionalGeneration(
             if i != self.config.talker_config.codec_eos_token_id
         ]
 
+    def get_mrope_input_positions(
+        self,
+        input_tokens: list[int],
+        mm_features: list[MultiModalFeatureSpec] | None = None,
+        **kwargs: object,
+    ) -> tuple[torch.Tensor, int]:
+        if self.model_stage == "thinker":
+            if mm_features is None:
+                msg = "Qwen3 Omni thinker get_mrope_input_positions requires mm_features"
+                raise ValueError(msg)
+            return self.thinker.get_mrope_input_positions(input_tokens, mm_features)
+        return MRotaryEmbedding.get_input_positions_tensor(input_tokens, **kwargs)
+
     def forward(
         self,
         input_ids: torch.Tensor,
@@ -330,22 +345,17 @@ class Qwen3OmniMoeForConditionalGeneration(
             if is_npu():
                 # TODO: remove this hack when NPU supports batched inputs properly
                 thinker_input_ids = input_ids[0] if input_ids is not None and _added_batch_dim else input_ids
-                thinker_positions = positions[0] if positions.ndim > 1 else positions
                 thinker_inputs_embeds = (
                     inputs_embeds[0] if inputs_embeds is not None and _added_batch_dim else inputs_embeds
                 )
             else:
                 thinker_input_ids = input_ids
-                thinker_positions = positions[0] if positions.ndim > 1 else positions
                 thinker_inputs_embeds = inputs_embeds
-            # thinker_input_ids = input_ids
-            # thinker_positions = positions[0] if positions.ndim > 1 else positions
-            # thinker_inputs_embeds = inputs_embeds
 
             # Run thinker
             text_hidden_states, captured_layer_dict = self.thinker(
                 input_ids=thinker_input_ids,
-                positions=thinker_positions,
+                positions=positions,
                 intermediate_tensors=intermediate_tensors,
                 inputs_embeds=thinker_inputs_embeds,
                 **capture_kwargs,

vllm_omni/worker/gpu_model_runner.py

@@ -77,6 +77,7 @@ class OmniGPUModelRunner(GPUModelRunner):
         if supports_mrope(self.model):
             req_state.mrope_positions, req_state.mrope_position_delta = self.model.get_mrope_input_positions(
                 req_state.prompt_token_ids,
+                mm_features=req_state.mm_features,
                 hf_config=self.model_config.hf_config,
                 image_grid_thw=image_grid_thw,
                 video_grid_thw=video_grid_thw,