[Feature] Enable use_audio_in_video for Qwen 3 Omni Online (#1198)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>

examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -308,19 +308,24 @@ def get_multi_audios_query(custom_prompt: str | None = None):
 
 def get_use_audio_in_video_query(
     video_path: str | None = None,
-    audio_path: str | None = None,
     custom_prompt: str | None = None,
 ):
+    """Query for use_audio_in_video mode.
+
+    When use_audio_in_video=True, audio is automatically extracted from the video
+    by the server. Do NOT send a separate audio_url - this would cause a mismatch
+    between the number of audio and video items.
+    """
     question = custom_prompt or (
         "Describe the content of the video in details, then convert what the baby say into text."
     )
     video_url = get_video_url_from_path(video_path)
-    audio_url = get_audio_url_from_path(audio_path)
+    # Note: audio is extracted from video automatically when use_audio_in_video=True
+    # Do not include a separate audio_url here
     return {
         "role": "user",
         "content": [
             {"type": "video_url", "video_url": {"url": video_url}},
-            {"type": "audio_url", "audio_url": {"url": audio_url}},
             {"type": "text", "text": question},
         ],
     }
@@ -397,7 +402,6 @@ def run_multimodal_generation(args) -> None:
     elif args.query_type == "use_audio_in_video":
         prompt = query_func(
             video_path=video_path,
-            audio_path=audio_path,
             custom_prompt=custom_prompt,
         )
     else:
@@ -484,7 +488,7 @@ def parse_args():
         "--query-type",
         "-q",
         type=str,
-        default="use_mixed_modalities",
+        default="use_audio_in_video",
         choices=query_map.keys(),
         help="Query type.",
     )

vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py

@@ -336,14 +336,19 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
                 mm_item_counts,
             )
         else:
-            if use_audio_in_video and "audio" in mm_prompt_updates:
+            if use_audio_in_video:
+                # When use_audio_in_video=True, audio is extracted from video and embedded
+                # in the video placeholder tokens. We should:
+                # 1. Filter out audio from prompt updates (audio has no separate placeholder)
+                # 2. Apply remaining updates (video, image, etc.)
+                # 3. Derive audio placeholders from video placeholders
                 filtered_updates = {k: v for k, v in mm_prompt_updates.items() if k != "audio"}
                 prompt_ids, mm_placeholders = self._apply_prompt_updates(
                     prompt_ids,
                     filtered_updates,
                 )
                 # Derive audio placeholders from video placeholders
-                mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates)
+                mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_item_counts)
             else:
                 prompt_ids, mm_placeholders = self._apply_prompt_updates(
                     prompt_ids,
@@ -508,18 +513,27 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
     def _derive_audio_from_video_placeholders(
         self,
         placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
-        mm_prompt_updates: MultiModalPromptUpdates,
+        mm_item_counts: Mapping[str, int],
     ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
         """
         Helper to derive audio placeholders from video placeholders when
         use_audio_in_video=True.
+
+        In use_audio_in_video mode, audio is extracted from video and embedded
+        within the video placeholder tokens. This function creates audio placeholder
+        info by extracting the audio token positions from video placeholders.
+
+        Args:
+            placeholders: Current placeholders (should contain "video")
+            mm_item_counts: Counts of multimodal items from mm_items.get_all_counts()
         """
         if "video" not in placeholders:
             return placeholders
 
         # Validate audio and video counts match
+        # In use_audio_in_video mode, audio count comes from mm_items (extracted from video)
         num_videos = len(placeholders["video"])
-        num_audios = len(mm_prompt_updates.get("audio", []))
+        num_audios = mm_item_counts.get("audio", 0)
         if num_audios != num_videos:
             raise ValueError(
                 f"use_audio_in_video requires equal number of audio and video items, got {num_audios=}, {num_videos=}"