From 04c55d0164e586373a2d3a3fdd8ab792b72017e8 Mon Sep 17 00:00:00 2001 From: nodeeeeee Date: Thu, 17 Sep 2026 02:39:13 +0800 Subject: [PATCH] Preserve audio boundary embeddings in Qwen2.5-Omni interleaved input (#7510) Signed-off-by: nodeeeeee --- .../test_qwen2_5_omni_expansion.py | 26 ++++++++++ .../qwen2_5_omni/test_per_video_audio_mask.py | 2 +- .../qwen2_5_omni/test_qwen2_5_omni_embed.py | 50 +++++++++++++++++++ .../qwen2_5_omni/qwen2_5_omni_thinker.py | 8 ++- 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py index 1e5e59aed..389b0b6ba 100644 --- a/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen2_5_omni_expansion.py @@ -96,6 +96,32 @@ def test_mix_to_text_audio_001(omni_server, online_client) -> None: online_client.send_omni_request(request_config) +@pytest.mark.slow +@pytest.mark.omni +@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stream", [False, True]) +def test_audio_in_video_boundaries(omni_server, online_client, stream: bool) -> None: + """Interleaved video/audio input must not treat audio boundaries as image placeholders.""" + # More than two seconds exercises multiple interleaved video/audio chunks. + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 128, embed_audio=True)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + content_text="Describe what you see and hear in this video in one sentence.", + ) + responses = online_client.send_omni_request( + { + "model": omni_server.model, + "messages": messages, + "stream": stream, + "modalities": ["text"], + "use_audio_in_video": True, + } + ) + assert responses[0].text_content + + @pytest.mark.slow @pytest.mark.omni @pytest.mark.parametrize("omni_server", test_params, indirect=True) diff --git a/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py b/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py index 54efc99e4..9ebad492b 100644 --- a/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py +++ b/tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py @@ -582,7 +582,7 @@ def test_get_video_use_audio_in_video_falls_back_to_prompt_updates_for_cache_hit def test_derive_audio_from_video_placeholders_only_pairs_true_videos(): - fake_self = _fake_processor() + fake_self = _fake_qwen2_prompt_processor() video_placeholders = [ PlaceholderFeaturesInfo( modality="video", diff --git a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py index 7b624ba11..0d7cd8175 100644 --- a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py +++ b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py @@ -331,6 +331,56 @@ class TestEmbedInputIds: "Interleaved: audio positions should get audio embeddings" ) + def test_interleaved_audio_boundaries_keep_text_embeddings(self, mocker: MockerFixture): + from vllm.multimodal.processing.processor import PlaceholderFeaturesInfo + + from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( + Qwen2_5OmniThinkerMultiModalProcessor, + ) + + processor = mocker.Mock() + processor.info.get_hf_processor.return_value = mocker.Mock(audio_token="audio", video_token="video") + processor.info.get_tokenizer.return_value.get_vocab.return_value = { + "audio": AUDIO_TOKEN_ID, + "video": VIDEO_TOKEN_ID, + } + # Real audio_bos/audio_eos IDs from the failing use_audio_in_video input. + tokens = [151647, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, 151648] + placeholder = PlaceholderFeaturesInfo( + modality="video", + item_idx=0, + start_idx=2, + tokens=tokens, + is_embed=torch.ones(len(tokens), dtype=torch.bool), + ) + placeholders = Qwen2_5OmniThinkerMultiModalProcessor._derive_audio_from_video_placeholders( + processor, + {"video": [placeholder]}, + {"audio": [[object()]]}, + [True], + ) + + input_ids = torch.tensor([TEXT_TOKEN_ID] * 2 + tokens + [TEXT_TOKEN_ID]) + is_multimodal = torch.zeros_like(input_ids, dtype=torch.bool) + for modality in ("audio", "video"): + derived = placeholders[modality][0].to_range() + is_multimodal[derived.offset : derived.offset + derived.length] |= derived.is_embed + + model, hidden = make_mock_model(mocker, thinker_cls=_omni_qwen2_5_thinker_cls()) + text_embeddings = model.get_language_model().embed_input_ids(input_ids) + result = model.embed_input_ids( + input_ids, + build_mm_embeds(audio_n=2, image_n=0, video_n=2, hidden=hidden), + is_multimodal=is_multimodal, + ) + + is_audio = input_ids == AUDIO_TOKEN_ID + is_video = input_ids == VIDEO_TOKEN_ID + assert torch.equal(is_multimodal, is_audio | is_video) + torch.testing.assert_close(result[is_audio], torch.full((2, hidden), 10.0)) + torch.testing.assert_close(result[is_video], torch.full((2, hidden), 30.0)) + torch.testing.assert_close(result[~(is_audio | is_video)], text_embeddings[~(is_audio | is_video)]) + # --------------------------------------------------------------------------- # Tests for merge_interleaved_embeddings helper diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index 263ffa2ab..b7b7d074c 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -758,6 +758,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( tokenizer = self.info.get_tokenizer() processor = self.info.get_hf_processor() audio_token_id = tokenizer.get_vocab()[processor.audio_token] + video_token_id = tokenizer.get_vocab()[processor.video_token] result_placeholders = dict(placeholders) audio_placeholders = [] @@ -765,7 +766,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor( audio_idx = 0 for video_idx, video_placeholder in enumerate(placeholders["video"]): - audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id + placeholder_tokens = torch.tensor(video_placeholder.tokens) + audio_is_embed = placeholder_tokens == audio_token_id + # Audio boundary tokens keep their text embeddings. + video_is_embed = placeholder_tokens == video_token_id if video_use_audio_in_video[video_idx]: audio_placeholder = PlaceholderFeaturesInfo( @@ -783,7 +787,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( item_idx=video_idx, start_idx=video_placeholder.start_idx, tokens=video_placeholder.tokens, - is_embed=~audio_is_embed, + is_embed=video_is_embed, ) video_placeholders.append(video_placeholder_with_mask)