[Qwen2.5-Omni] Avoid redundant embedding computation (#7477)

Signed-off-by: nodeeeeee <zhangkai.nodeee@gmail.com>
This commit is contained in:
nodeeeeee authored and GitHub committed 2026-09-14 10:02:11 -07:00
1 parent ea1a416a17
commit 909b121e0a
5 files changed
+412 -13

No files matched your search

@@ -11,6 +11,7 @@ Regression test for: https://github.com/vllm-project/vllm/issues/34506
"""
import functools
from types import SimpleNamespace
import pytest
import torch
@@ -134,16 +135,15 @@ def make_mock_model(mocker: MockerFixture, hidden: int = 8, thinker_cls=None):
that has enough structure to run embed_input_ids.
``thinker_cls`` selects whose ``embed_input_ids`` the mock dispatches to.
It defaults to upstream's. Only the interleaved case passes omni's override:
omni's non-interleaved path falls through to a zero-arg ``super()`` call,
which raises TypeError when ``self`` is a Mock rather than an instance.
It defaults to upstream's. Use the selected class as the mock spec so
zero-argument ``super()`` in omni's override resolves correctly.
"""
m = _qwen2_5_omni_thinker_mod()
Qwen2_5OmniThinkerForConditionalGeneration = m.Qwen2_5OmniThinkerForConditionalGeneration
if thinker_cls is None:
thinker_cls = Qwen2_5OmniThinkerForConditionalGeneration
model = mocker.Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration)
model = mocker.Mock(spec=thinker_cls)
# Config with token IDs
cfg = mocker.Mock()
@@ -157,8 +157,7 @@ def make_mock_model(mocker: MockerFixture, hidden: int = 8, thinker_cls=None):
# view with shared memory, which masked_scatter_ cannot handle).
return ids.float().unsqueeze(-1).expand(-1, hidden).clone()
lang_model = mocker.Mock()
lang_model.embed_input_ids = fake_lm_embed
lang_model = SimpleNamespace(embed_input_ids=fake_lm_embed)
model.get_language_model = mocker.Mock(return_value=lang_model)
from vllm.model_executor.models.interfaces import SupportsMultiModal
@@ -271,6 +270,27 @@ class TestEmbedInputIds:
"Text positions should keep text embeddings"
)
@pytest.mark.parametrize(
"audio_n,image_n,video_n",
[(5, 0, 0), (0, 4, 0), (0, 0, 6), (5, 4, 6)],
ids=["audio", "image", "video", "mixed"],
)
def test_non_interleaved_multimodal_single_embedding(self, mocker: MockerFixture, audio_n, image_n, video_n):
"""Omni's regular path embeds once through super() and preserves placement."""
hidden = 8
input_ids, is_multimodal = make_token_seq(audio_n, image_n, video_n)
mm_embeds = build_mm_embeds(audio_n, image_n, video_n, hidden)
model, _ = make_mock_model(mocker, hidden, thinker_cls=_omni_qwen2_5_thinker_cls())
embed = mocker.spy(model.get_language_model(), "embed_input_ids")
result = model.embed_input_ids(input_ids, mm_embeds, is_multimodal=is_multimodal)
embed.assert_called_once()
expected = input_ids.float().unsqueeze(-1).expand(-1, hidden).clone()
for token_id, value in ((AUDIO_TOKEN_ID, 10.0), (IMAGE_TOKEN_ID, 20.0), (VIDEO_TOKEN_ID, 30.0)):
expected[input_ids == token_id] = value
assert torch.equal(result, expected)
def test_interleaved_use_audio_in_video(self, mocker: MockerFixture):
"""
Interleaved (use_audio_in_video): video chunks interleaved with audio.
@@ -296,7 +316,9 @@ class TestEmbedInputIds:
]
model, _ = make_mock_model(mocker, hidden, thinker_cls=_omni_qwen2_5_thinker_cls())
embed = mocker.spy(model.get_language_model(), "embed_input_ids")
result = model.embed_input_ids(input_ids, mm_embeds, is_multimodal=is_multimodal)
embed.assert_called_once()
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]