Preserve audio boundary embeddings in Qwen2.5-Omni interleaved input (#7510)

Signed-off-by: nodeeeeee <zhangkai.nodeee@gmail.com>

tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py

@@ -582,7 +582,7 @@ def test_get_video_use_audio_in_video_falls_back_to_prompt_updates_for_cache_hit
 
 
 def test_derive_audio_from_video_placeholders_only_pairs_true_videos():
-    fake_self = _fake_processor()
+    fake_self = _fake_qwen2_prompt_processor()
     video_placeholders = [
         PlaceholderFeaturesInfo(
             modality="video",

tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py

@@ -331,6 +331,56 @@ class TestEmbedInputIds:
             "Interleaved: audio positions should get audio embeddings"
         )
 
+    def test_interleaved_audio_boundaries_keep_text_embeddings(self, mocker: MockerFixture):
+        from vllm.multimodal.processing.processor import PlaceholderFeaturesInfo
+
+        from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
+            Qwen2_5OmniThinkerMultiModalProcessor,
+        )
+
+        processor = mocker.Mock()
+        processor.info.get_hf_processor.return_value = mocker.Mock(audio_token="audio", video_token="video")
+        processor.info.get_tokenizer.return_value.get_vocab.return_value = {
+            "audio": AUDIO_TOKEN_ID,
+            "video": VIDEO_TOKEN_ID,
+        }
+        # Real audio_bos/audio_eos IDs from the failing use_audio_in_video input.
+        tokens = [151647, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, 151648]
+        placeholder = PlaceholderFeaturesInfo(
+            modality="video",
+            item_idx=0,
+            start_idx=2,
+            tokens=tokens,
+            is_embed=torch.ones(len(tokens), dtype=torch.bool),
+        )
+        placeholders = Qwen2_5OmniThinkerMultiModalProcessor._derive_audio_from_video_placeholders(
+            processor,
+            {"video": [placeholder]},
+            {"audio": [[object()]]},
+            [True],
+        )
+
+        input_ids = torch.tensor([TEXT_TOKEN_ID] * 2 + tokens + [TEXT_TOKEN_ID])
+        is_multimodal = torch.zeros_like(input_ids, dtype=torch.bool)
+        for modality in ("audio", "video"):
+            derived = placeholders[modality][0].to_range()
+            is_multimodal[derived.offset : derived.offset + derived.length] |= derived.is_embed
+
+        model, hidden = make_mock_model(mocker, thinker_cls=_omni_qwen2_5_thinker_cls())
+        text_embeddings = model.get_language_model().embed_input_ids(input_ids)
+        result = model.embed_input_ids(
+            input_ids,
+            build_mm_embeds(audio_n=2, image_n=0, video_n=2, hidden=hidden),
+            is_multimodal=is_multimodal,
+        )
+
+        is_audio = input_ids == AUDIO_TOKEN_ID
+        is_video = input_ids == VIDEO_TOKEN_ID
+        assert torch.equal(is_multimodal, is_audio | is_video)
+        torch.testing.assert_close(result[is_audio], torch.full((2, hidden), 10.0))
+        torch.testing.assert_close(result[is_video], torch.full((2, hidden), 30.0))
+        torch.testing.assert_close(result[~(is_audio | is_video)], text_embeddings[~(is_audio | is_video)])
+
 
 # ---------------------------------------------------------------------------
 # Tests for merge_interleaved_embeddings helper