From d7678e2bb30df56aad929e290f194b930f2b73b1 Mon Sep 17 00:00:00 2001 From: eval <74645252+eval-dev@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:41:42 -0700 Subject: [PATCH] [Bugfix][MiniCPM-o] Fix the audio_embeds input path (#5730) Signed-off-by: eval-dev <0xe5bca0@gmail.com> Signed-off-by: eval <74645252+eval-dev@users.noreply.github.com> --- .../minicpmo_4_5/test_audio_embeds_input.py | 138 ++++++++++++++++++ .../minicpmo_4_5/minicpmo_4_5_omni_llm.py | 5 +- 2 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 tests/model_executor/models/minicpmo_4_5/test_audio_embeds_input.py diff --git a/tests/model_executor/models/minicpmo_4_5/test_audio_embeds_input.py b/tests/model_executor/models/minicpmo_4_5/test_audio_embeds_input.py new file mode 100644 index 000000000..d25b58e48 --- /dev/null +++ b/tests/model_executor/models/minicpmo_4_5/test_audio_embeds_input.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression tests for the MiniCPM-o 4.5 ``audio_embeds`` input path. + +Two defects made the OpenAI ``audio_embeds`` content part unusable with this model: + +1. ``MiniCPMOAudioEmbeddingItems`` registered itself under ``modality="image"`` + (copy-pasted from ``MiniCPMVImageEmbeddingItems``), so every request failed with + ``Modality 'image' not found. Available modalities: {'audio'}``. +2. ``_get_prompt_updates`` sized the placeholder from + ``sum(map(len, single_audio_embeds))``. ``MiniCPMOAudioEmbeddingInputs`` declares + ``TensorShape("bn", "s", "h")``, so one item is ``(s, h)`` and that expression + iterates rows and sums the hidden size -- ``s * h`` instead of ``s``. A one-second + slice of shape ``(10, 4096)`` asked for 40960 placeholder tokens instead of 10, + pushing every real request past ``max_model_len``. +""" + +from __future__ import annotations + +import math +from types import SimpleNamespace + +import pytest +import torch +from vllm.multimodal.parse import MultiModalDataItems + +from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import ( + MiniCPMO45OmniLLMMultiModalProcessor, + MiniCPMO45OmniLLMProcessingInfo, + MiniCPMOAudioEmbeddingItems, + _minicpmo_field_config, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +POOL_STEP = 5 +HIDDEN_SIZE = 4096 +#: Audio embedding vectors produced per second: 100 mel -> 50 cnn -> (50-5)//5+1. +EMBEDS_PER_SECOND = 10 + + +class _FakeProcessor: + """Minimal processor surface used by ``get_audio_placeholder``.""" + + def __init__(self) -> None: + self.pool_step = POOL_STEP + self.audio_processor = SimpleNamespace(hop_length=160) + self.image_processor = SimpleNamespace(mean=[0.0], std=[1.0]) + + def get_audio_placeholder( + self, + audio_lens: int, + chunk_input: bool = True, + chunk_length: int = 1, + ) -> str: + del chunk_input, chunk_length + feature_lens = math.ceil(audio_lens / self.audio_processor.hop_length) + cnn_feature_lens = (feature_lens - 1) // 2 + 1 + output_lens = (cnn_feature_lens - self.pool_step) // self.pool_step + 1 + return "" * output_lens + + +class _IdentityTokenizer: + """``_get_prompt_updates`` round-trips the image/video patterns through the + tokenizer to detect encode/decode drift; an identity pair keeps that a no-op.""" + + def encode(self, text: str, add_special_tokens: bool = True) -> str: + del add_special_tokens + return text + + def decode(self, tokens: str) -> str: + return tokens + + +def _processing_info() -> MiniCPMO45OmniLLMProcessingInfo: + processor = _FakeProcessor() + tokenizer = _IdentityTokenizer() + info = object.__new__(MiniCPMO45OmniLLMProcessingInfo) + info.ctx = SimpleNamespace( + tokenizer=tokenizer, + get_tokenizer=lambda: tokenizer, + get_hf_config=lambda: SimpleNamespace(audio_pool_step=POOL_STEP), + get_hf_processor=lambda **kwargs: processor, + ) + return info + + +def _audio_placeholder_for(single_audio_embeds: torch.Tensor) -> str: + """Run the real prompt-update path and return the audio placeholder text. + + Goes through ``_get_prompt_updates`` rather than recomputing the length, so the + ``sum(map(len, ...))`` regression is actually exercised. + """ + info = _processing_info() + proc = object.__new__(MiniCPMO45OmniLLMMultiModalProcessor) + proc.info = info + + items = MiniCPMOAudioEmbeddingItems( + {"audio_embeds": single_audio_embeds.unsqueeze(0)}, + fields_factory=_minicpmo_field_config, + ) + mm_items = MultiModalDataItems({"audio": items}) + + updates = proc._get_prompt_updates(mm_items, {}, {}) + audio_update = next(u for u in updates if u.modality == "audio") + return audio_update.content(0).full + + +def test_audio_embedding_items_use_the_audio_modality() -> None: + """Registering under ``image`` makes every ``audio_embeds`` request fail.""" + items = MiniCPMOAudioEmbeddingItems( + {"audio_embeds": torch.zeros(1, EMBEDS_PER_SECOND, HIDDEN_SIZE)}, + fields_factory=_minicpmo_field_config, + ) + + assert items.modality == "audio" + + +@pytest.mark.parametrize("num_seconds", [1, 3, 30]) +def test_placeholder_length_matches_embedding_count(num_seconds: int) -> None: + """An ``(s, h)`` item must yield exactly ``s`` placeholder tokens.""" + num_embeds = num_seconds * EMBEDS_PER_SECOND + + placeholder = _audio_placeholder_for(torch.zeros(num_embeds, HIDDEN_SIZE)) + + assert placeholder.count("") == num_embeds + + +def test_placeholder_does_not_scale_with_hidden_size() -> None: + """The count must depend on ``s`` alone, never on ``h``. + + ``sum(map(len, ...))`` returned ``s * h``, so the placeholder grew with the model's + hidden size. Two items with the same ``s`` but different ``h`` must agree. + """ + narrow = _audio_placeholder_for(torch.zeros(EMBEDS_PER_SECOND, 8)) + wide = _audio_placeholder_for(torch.zeros(EMBEDS_PER_SECOND, HIDDEN_SIZE)) + + assert narrow.count("") == wide.count("") == EMBEDS_PER_SECOND diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index f6c9b1fa9..9dfff5589 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -3310,7 +3310,7 @@ class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems): ) -> None: super().__init__( data, - modality="image", + modality="audio", required_fields={"audio_embeds"}, fields_factory=fields_factory, ) @@ -3733,7 +3733,8 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn if isinstance(audios, MiniCPMOAudioEmbeddingItems): single_audio_embeds = audios.get(item_idx)["audio_embeds"] - audio_len = self.info.get_audio_len_by_num_chunks(sum(map(len, single_audio_embeds))) + # One item is ``(s, h)``, so its leading dim is the audio embedding count. + audio_len = self.info.get_audio_len_by_num_chunks(len(single_audio_embeds)) else: audio_len = audios.get_audio_length(item_idx)