Preserve audio boundary embeddings in Qwen2.5-Omni interleaved input (#7510)

Signed-off-by: nodeeeeee <zhangkai.nodeee@gmail.com>

tests/e2e/online_serving/test_qwen2_5_omni_expansion.py

@@ -96,6 +96,32 @@ def test_mix_to_text_audio_001(omni_server, online_client) -> None:
     online_client.send_omni_request(request_config)
 
 
+@pytest.mark.slow
+@pytest.mark.omni
+@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+@pytest.mark.parametrize("stream", [False, True])
+def test_audio_in_video_boundaries(omni_server, online_client, stream: bool) -> None:
+    """Interleaved video/audio input must not treat audio boundaries as image placeholders."""
+    # More than two seconds exercises multiple interleaved video/audio chunks.
+    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 128, embed_audio=True)['base64']}"
+    messages = dummy_messages_from_mix_data(
+        system_prompt=get_system_prompt(),
+        video_data_url=video_data_url,
+        content_text="Describe what you see and hear in this video in one sentence.",
+    )
+    responses = online_client.send_omni_request(
+        {
+            "model": omni_server.model,
+            "messages": messages,
+            "stream": stream,
+            "modalities": ["text"],
+            "use_audio_in_video": True,
+        }
+    )
+    assert responses[0].text_content
+
+
 @pytest.mark.slow
 @pytest.mark.omni
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)

tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py

@@ -582,7 +582,7 @@ def test_get_video_use_audio_in_video_falls_back_to_prompt_updates_for_cache_hit
 
 
 def test_derive_audio_from_video_placeholders_only_pairs_true_videos():
-    fake_self = _fake_processor()
+    fake_self = _fake_qwen2_prompt_processor()
     video_placeholders = [
         PlaceholderFeaturesInfo(
             modality="video",

tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py

@@ -331,6 +331,56 @@ class TestEmbedInputIds:
             "Interleaved: audio positions should get audio embeddings"
         )
 
+    def test_interleaved_audio_boundaries_keep_text_embeddings(self, mocker: MockerFixture):
+        from vllm.multimodal.processing.processor import PlaceholderFeaturesInfo
+
+        from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
+            Qwen2_5OmniThinkerMultiModalProcessor,
+        )
+
+        processor = mocker.Mock()
+        processor.info.get_hf_processor.return_value = mocker.Mock(audio_token="audio", video_token="video")
+        processor.info.get_tokenizer.return_value.get_vocab.return_value = {
+            "audio": AUDIO_TOKEN_ID,
+            "video": VIDEO_TOKEN_ID,
+        }
+        # Real audio_bos/audio_eos IDs from the failing use_audio_in_video input.
+        tokens = [151647, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, 151648]
+        placeholder = PlaceholderFeaturesInfo(
+            modality="video",
+            item_idx=0,
+            start_idx=2,
+            tokens=tokens,
+            is_embed=torch.ones(len(tokens), dtype=torch.bool),
+        )
+        placeholders = Qwen2_5OmniThinkerMultiModalProcessor._derive_audio_from_video_placeholders(
+            processor,
+            {"video": [placeholder]},
+            {"audio": [[object()]]},
+            [True],
+        )
+
+        input_ids = torch.tensor([TEXT_TOKEN_ID] * 2 + tokens + [TEXT_TOKEN_ID])
+        is_multimodal = torch.zeros_like(input_ids, dtype=torch.bool)
+        for modality in ("audio", "video"):
+            derived = placeholders[modality][0].to_range()
+            is_multimodal[derived.offset : derived.offset + derived.length] |= derived.is_embed
+
+        model, hidden = make_mock_model(mocker, thinker_cls=_omni_qwen2_5_thinker_cls())
+        text_embeddings = model.get_language_model().embed_input_ids(input_ids)
+        result = model.embed_input_ids(
+            input_ids,
+            build_mm_embeds(audio_n=2, image_n=0, video_n=2, hidden=hidden),
+            is_multimodal=is_multimodal,
+        )
+
+        is_audio = input_ids == AUDIO_TOKEN_ID
+        is_video = input_ids == VIDEO_TOKEN_ID
+        assert torch.equal(is_multimodal, is_audio | is_video)
+        torch.testing.assert_close(result[is_audio], torch.full((2, hidden), 10.0))
+        torch.testing.assert_close(result[is_video], torch.full((2, hidden), 30.0))
+        torch.testing.assert_close(result[~(is_audio | is_video)], text_embeddings[~(is_audio | is_video)])
+
 
 # ---------------------------------------------------------------------------
 # Tests for merge_interleaved_embeddings helper

vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py

@@ -758,6 +758,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
         tokenizer = self.info.get_tokenizer()
         processor = self.info.get_hf_processor()
         audio_token_id = tokenizer.get_vocab()[processor.audio_token]
+        video_token_id = tokenizer.get_vocab()[processor.video_token]
 
         result_placeholders = dict(placeholders)
         audio_placeholders = []
@@ -765,7 +766,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
 
         audio_idx = 0
         for video_idx, video_placeholder in enumerate(placeholders["video"]):
-            audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id
+            placeholder_tokens = torch.tensor(video_placeholder.tokens)
+            audio_is_embed = placeholder_tokens == audio_token_id
+            # Audio boundary tokens keep their text embeddings.
+            video_is_embed = placeholder_tokens == video_token_id
 
             if video_use_audio_in_video[video_idx]:
                 audio_placeholder = PlaceholderFeaturesInfo(
@@ -783,7 +787,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
                 item_idx=video_idx,
                 start_idx=video_placeholder.start_idx,
                 tokens=video_placeholder.tokens,
-                is_embed=~audio_is_embed,
+                is_embed=video_is_embed,
             )
             video_placeholders.append(video_placeholder_with_mask)