diff --git a/examples/offline_inference/qwen2_5_omni/end2end.py b/examples/offline_inference/qwen2_5_omni/end2end.py index bbeef32b0..91a7305ec 100644 --- a/examples/offline_inference/qwen2_5_omni/end2end.py +++ b/examples/offline_inference/qwen2_5_omni/end2end.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """ This example shows how to use vLLM-Omni for running offline inference with the correct prompt format on Qwen2.5-Omni @@ -137,7 +137,9 @@ def get_use_audio_in_video_query( question = "Describe the content of the video, then convert what the baby say into text." prompt = ( f"<|im_start|>system\n{default_system}<|im_end|>\n" - "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|><|audio_bos|><|AUDIO|><|audio_eos|>" + # With use_audio_in_video=True the processor interleaves the audio into + # the video placeholder, so no separate <|AUDIO|> placeholder is used. + "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>" f"{question}<|im_end|>\n" f"<|im_start|>assistant\n" ) diff --git a/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py b/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py index 9ebad492b..699acc951 100644 --- a/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py +++ b/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py @@ -619,6 +619,39 @@ def test_derive_audio_from_video_placeholders_only_pairs_true_videos(): assert result["video"][1].is_embed.tolist() == [True, True] +def _interleaved_video_placeholder() -> PlaceholderFeaturesInfo: + return PlaceholderFeaturesInfo( + modality="video", + item_idx=0, + start_idx=1, + tokens=[AUDIO_BOS_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, AUDIO_EOS_TOKEN_ID], + is_embed=torch.tensor([False, True, False, False]), + ) + + +def test_reject_audio_tokens_outside_videos_accepts_interleaved_audio(): + prompt_ids = [0, AUDIO_BOS_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, AUDIO_EOS_TOKEN_ID, 0] + + Qwen2_5OmniThinkerMultiModalProcessor._reject_audio_tokens_outside_videos( + _fake_qwen2_prompt_processor(), + prompt_ids, + {"video": [_interleaved_video_placeholder()]}, + ) + + +def test_reject_audio_tokens_outside_videos_rejects_standalone_audio_placeholder(): + # A separate <|AUDIO|> after the video is left unexpanded when + # use_audio_in_video=True and has no audio features behind it. + prompt_ids = [0, AUDIO_BOS_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, AUDIO_EOS_TOKEN_ID, AUDIO_TOKEN_ID, 0] + + with pytest.raises(ValueError, match="remove the separate <\\|AUDIO\\|> placeholder"): + Qwen2_5OmniThinkerMultiModalProcessor._reject_audio_tokens_outside_videos( + _fake_qwen2_prompt_processor(), + prompt_ids, + {"video": [_interleaved_video_placeholder()]}, + ) + + def test_qwen3_processor_inherits_vllm_omni_per_video_helpers(): from vllm.model_executor.models.qwen3_omni_moe_thinker import ( Qwen3OmniMoeThinkerMultiModalProcessor as UpstreamQwen3Processor, diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index b7b7d074c..1f89a37c4 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -795,6 +795,27 @@ class Qwen2_5OmniThinkerMultiModalProcessor( result_placeholders["video"] = video_placeholders return result_placeholders + def _reject_audio_tokens_outside_videos( + self, + prompt_ids: list[int], + placeholders: Mapping[str, list[PlaceholderFeaturesInfo]], + ) -> None: + # With use_audio_in_video, every audio item is interleaved into a video + # placeholder and standalone audio updates are skipped. A leftover + # <|AUDIO|> would reach the engine with no audio features behind it and + # crash MRoPE position computation, so reject the request here instead. + tokenizer = self.info.get_tokenizer() + audio_token_id = tokenizer.get_vocab()[self.info.get_hf_processor().audio_token] + in_video = [False] * len(prompt_ids) + for placeholder in placeholders.get("video", []): + end = placeholder.start_idx + len(placeholder.tokens) + in_video[placeholder.start_idx : end] = [True] * (end - placeholder.start_idx) + if any(tok == audio_token_id and not in_video[i] for i, tok in enumerate(prompt_ids)): + raise ValueError( + "use_audio_in_video=True interleaves each audio into its video " + "placeholder; remove the separate <|AUDIO|> placeholder from the prompt." + ) + def _maybe_apply_prompt_updates( self, mm_items: MultiModalDataItems, @@ -820,6 +841,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( mm_prompt_updates, video_use_audio_in_video, ) + self._reject_audio_tokens_outside_videos(prompt_ids, mm_placeholders) else: prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids,