[Bugfix] Reject stray <|AUDIO|> placeholder with use_audio_in_video in Qwen2.5-Omni (#7959)

Signed-off-by: RunguoLi <li19107254665@gmail.com>

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """
 This example shows how to use vLLM-Omni for running offline inference
 with the correct prompt format on Qwen2.5-Omni
@@ -137,7 +137,9 @@ def get_use_audio_in_video_query(
     question = "Describe the content of the video, then convert what the baby say into text."
     prompt = (
         f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|><|audio_bos|><|AUDIO|><|audio_eos|>"
+        # With use_audio_in_video=True the processor interleaves the audio into
+        # the video placeholder, so no separate <|AUDIO|> placeholder is used.
+        "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>"
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )

tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py

@@ -619,6 +619,39 @@ def test_derive_audio_from_video_placeholders_only_pairs_true_videos():
     assert result["video"][1].is_embed.tolist() == [True, True]
 
 
+def _interleaved_video_placeholder() -> PlaceholderFeaturesInfo:
+    return PlaceholderFeaturesInfo(
+        modality="video",
+        item_idx=0,
+        start_idx=1,
+        tokens=[AUDIO_BOS_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, AUDIO_EOS_TOKEN_ID],
+        is_embed=torch.tensor([False, True, False, False]),
+    )
+
+
+def test_reject_audio_tokens_outside_videos_accepts_interleaved_audio():
+    prompt_ids = [0, AUDIO_BOS_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, AUDIO_EOS_TOKEN_ID, 0]
+
+    Qwen2_5OmniThinkerMultiModalProcessor._reject_audio_tokens_outside_videos(
+        _fake_qwen2_prompt_processor(),
+        prompt_ids,
+        {"video": [_interleaved_video_placeholder()]},
+    )
+
+
+def test_reject_audio_tokens_outside_videos_rejects_standalone_audio_placeholder():
+    # A separate <|AUDIO|> after the video is left unexpanded when
+    # use_audio_in_video=True and has no audio features behind it.
+    prompt_ids = [0, AUDIO_BOS_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, AUDIO_EOS_TOKEN_ID, AUDIO_TOKEN_ID, 0]
+
+    with pytest.raises(ValueError, match="remove the separate <\\|AUDIO\\|> placeholder"):
+        Qwen2_5OmniThinkerMultiModalProcessor._reject_audio_tokens_outside_videos(
+            _fake_qwen2_prompt_processor(),
+            prompt_ids,
+            {"video": [_interleaved_video_placeholder()]},
+        )
+
+
 def test_qwen3_processor_inherits_vllm_omni_per_video_helpers():
     from vllm.model_executor.models.qwen3_omni_moe_thinker import (
         Qwen3OmniMoeThinkerMultiModalProcessor as UpstreamQwen3Processor,

vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py

@@ -795,6 +795,27 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
         result_placeholders["video"] = video_placeholders
         return result_placeholders
 
+    def _reject_audio_tokens_outside_videos(
+        self,
+        prompt_ids: list[int],
+        placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
+    ) -> None:
+        # With use_audio_in_video, every audio item is interleaved into a video
+        # placeholder and standalone audio updates are skipped. A leftover
+        # <|AUDIO|> would reach the engine with no audio features behind it and
+        # crash MRoPE position computation, so reject the request here instead.
+        tokenizer = self.info.get_tokenizer()
+        audio_token_id = tokenizer.get_vocab()[self.info.get_hf_processor().audio_token]
+        in_video = [False] * len(prompt_ids)
+        for placeholder in placeholders.get("video", []):
+            end = placeholder.start_idx + len(placeholder.tokens)
+            in_video[placeholder.start_idx : end] = [True] * (end - placeholder.start_idx)
+        if any(tok == audio_token_id and not in_video[i] for i, tok in enumerate(prompt_ids)):
+            raise ValueError(
+                "use_audio_in_video=True interleaves each audio into its video "
+                "placeholder; remove the separate <|AUDIO|> placeholder from the prompt."
+            )
+
     def _maybe_apply_prompt_updates(
         self,
         mm_items: MultiModalDataItems,
@@ -820,6 +841,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
                 mm_prompt_updates,
                 video_use_audio_in_video,
             )
+            self._reject_audio_tokens_outside_videos(prompt_ids, mm_placeholders)
         else:
             prompt_ids, mm_placeholders = self._apply_prompt_updates(
                 prompt_ids,