[Bugfix] MiniCPMo-4.5 bugfix (#5233)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Co-authored-by: Guanzhe (Leo) Huang <116847811+leohuang257@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

tests/model_executor/models/minicpmo_4_5/test_long_audio_processing.py

@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Regression tests for MiniCPM-o 4.5 >30s audio preprocessing.
+
+Covers:
+  - ``_minicpmo_field_config``: <=30s audio keeps the batched layout
+  - >30s audio switches to ``flat`` with one slice per source audio
+  - mixed long + short audios group chunks by source audio
+  - no audio inputs -> batched (default path unchanged)
+  - ``process_audios``: >30s audio unpads per chunk without ``TypeError``
+  - multiple >30s audios unpad in chunk order
+  - <=30s audio unpadding byte-identical to the pre-fix behavior
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import torch
+from vllm.multimodal.inputs import (
+    MultiModalBatchedField,
+    MultiModalFlatField,
+)
+
+from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
+    MiniCPMO45OmniLLMMultiModalProcessor,
+    MiniCPMOMultiModalDataParser,
+    _minicpmo_field_config,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+_SAMPLE_RATE = 16_000
+_FEAT_DIM = 128
+# Whisper pads every 30s chunk to 3000 mel frames.
+_CHUNK_FRAMES = 3000
+
+
+class TestMiniCPMOFieldConfig:
+    def test_short_audios_stay_batched(self) -> None:
+        # two <=30s audios: one feature entry + one 1-element lens per audio
+        hf_inputs = {
+            "audio_features": [torch.zeros(_FEAT_DIM, _CHUNK_FRAMES)] * 2,
+            "audio_feature_lens": [torch.tensor([980]), torch.tensor([1500])],
+        }
+
+        config = _minicpmo_field_config(hf_inputs)
+
+        assert isinstance(config["audio_features"].field, MultiModalBatchedField)
+
+    def test_long_audio_groups_chunks_per_audio(self) -> None:
+        # one 45s audio -> 2 chunks but a single lens tensor
+        hf_inputs = {
+            "audio_features": [torch.zeros(_FEAT_DIM, _CHUNK_FRAMES)] * 2,
+            "audio_feature_lens": [torch.tensor([_CHUNK_FRAMES, 1500])],
+        }
+
+        config = _minicpmo_field_config(hf_inputs)
+
+        field = config["audio_features"].field
+        assert isinstance(field, MultiModalFlatField)
+        assert field.slices == [slice(0, 2)]
+
+    def test_mixed_long_and_short_audios(self) -> None:
+        # 45s audio (2 chunks) + 10s audio (1 chunk)
+        hf_inputs = {
+            "audio_features": [torch.zeros(_FEAT_DIM, _CHUNK_FRAMES)] * 3,
+            "audio_feature_lens": [
+                torch.tensor([_CHUNK_FRAMES, 1500]),
+                torch.tensor([980]),
+            ],
+        }
+
+        config = _minicpmo_field_config(hf_inputs)
+
+        field = config["audio_features"].field
+        assert isinstance(field, MultiModalFlatField)
+        assert field.slices == [slice(0, 2), slice(2, 3)]
+
+    def test_no_audio_inputs_stay_batched(self) -> None:
+        config = _minicpmo_field_config({})
+
+        assert isinstance(config["audio_features"].field, MultiModalBatchedField)
+
+
+def _make_processor(
+    fake_hf_outputs: dict[str, list[torch.Tensor]],
+) -> MiniCPMO45OmniLLMMultiModalProcessor:
+    """Build a processor without a checkpoint: only what ``process_audios``
+    touches, with the HF processor call returning canned Whisper-style output."""
+    processor = object.__new__(MiniCPMO45OmniLLMMultiModalProcessor)
+    processor.info = SimpleNamespace(audio_pattern="(<audio>./</audio>)")
+    processor.data_parser = MiniCPMOMultiModalDataParser(target_sr=_SAMPLE_RATE)
+
+    def fake_base_call_hf_processor(prompts, mm_data, mm_kwargs, tok_kwargs, *, out_keys):
+        return {key: fake_hf_outputs[key] for key in out_keys}
+
+    processor._base_call_hf_processor = fake_base_call_hf_processor
+    return processor
+
+
+def _audio_seconds(seconds: float) -> np.ndarray:
+    return np.zeros(int(seconds * _SAMPLE_RATE), dtype=np.float32)
+
+
+class TestProcessAudiosUnpadding:
+    def test_long_audio_unpads_per_chunk(self) -> None:
+        # 45s audio: 2 padded chunks, one lens tensor -> pre-fix TypeError
+        chunk_lens = [_CHUNK_FRAMES, 1500]
+        fake_outputs = {
+            "audio_features": [torch.randn(_FEAT_DIM, _CHUNK_FRAMES) for _ in chunk_lens],
+            "audio_feature_lens": [torch.tensor(chunk_lens)],
+        }
+        processor = _make_processor(fake_outputs)
+
+        result = processor.process_audios({"audios": [_audio_seconds(45)]}, {}, {})
+
+        features = result["audio_features"]
+        assert [feat.shape[-1] for feat in features] == chunk_lens
+        for feat, padded, length in zip(features, fake_outputs["audio_features"], chunk_lens):
+            assert torch.equal(feat, padded[:, :length])
+
+    def test_multiple_long_audios(self) -> None:
+        # two >30s audios: 4 chunks total, 2-element lens tensor each
+        fake_outputs = {
+            "audio_features": [torch.randn(_FEAT_DIM, _CHUNK_FRAMES) for _ in range(4)],
+            "audio_feature_lens": [
+                torch.tensor([_CHUNK_FRAMES, 1200]),
+                torch.tensor([_CHUNK_FRAMES, 800]),
+            ],
+        }
+        processor = _make_processor(fake_outputs)
+
+        result = processor.process_audios({"audios": [_audio_seconds(42), _audio_seconds(38)]}, {}, {})
+
+        widths = [feat.shape[-1] for feat in result["audio_features"]]
+        assert widths == [_CHUNK_FRAMES, 1200, _CHUNK_FRAMES, 800]
+
+    def test_short_audio_unchanged(self) -> None:
+        # <=30s audio: single-chunk unpadding identical to pre-fix behavior
+        fake_outputs = {
+            "audio_features": [torch.randn(_FEAT_DIM, _CHUNK_FRAMES)],
+            "audio_feature_lens": [torch.tensor([980])],
+        }
+        processor = _make_processor(fake_outputs)
+
+        result = processor.process_audios({"audios": [_audio_seconds(9.8)]}, {}, {})
+
+        features = result["audio_features"]
+        assert len(features) == 1
+        assert features[0].shape == (_FEAT_DIM, 980)
+        assert torch.equal(features[0], fake_outputs["audio_features"][0][:, :980])

vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py

@@ -3175,9 +3175,33 @@ class MiniCPMOAudioEmbeddingItems(DictEmbeddingItems):
 
 
 def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
+    audio_features = hf_inputs.get("audio_features")
+    audio_feature_lens = hf_inputs.get("audio_feature_lens")
+    audio_features_cfg = MultiModalFieldConfig.batched("audio")
+
+    if audio_features is not None and audio_feature_lens is not None:
+        num_features = len(audio_features)
+        num_audios = len(audio_feature_lens)
+
+        if num_features > num_audios:
+            chunks_per_audio = [lens.numel() if isinstance(lens, torch.Tensor) else 1 for lens in audio_feature_lens]
+            if sum(chunks_per_audio) != num_features:
+                chunks_per_audio = [
+                    max(int((lens != 0).sum()), 1) if isinstance(lens, torch.Tensor) else 1
+                    for lens in audio_feature_lens
+                ]
+
+            slice_idxs = [0]
+            for num_chunks in chunks_per_audio:
+                slice_idxs.append(slice_idxs[-1] + num_chunks)
+            audio_features_cfg = MultiModalFieldConfig.flat(
+                "audio",
+                [slice(slice_idxs[i], slice_idxs[i + 1]) for i in range(len(chunks_per_audio))],
+            )
+
     return dict(
         **_minicpmv_field_config(hf_inputs),
-        audio_features=MultiModalFieldConfig.batched("audio"),
+        audio_features=audio_features_cfg,
         audio_feature_lens=MultiModalFieldConfig.batched("audio"),
         audio_embeds=MultiModalFieldConfig.batched("audio"),
     )
@@ -3369,11 +3393,17 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
                 out_keys={"audio_features", "audio_feature_lens"},
             )
 
+            flat_feature_lens: list[int] = []
+            for lens in audio_inputs["audio_feature_lens"]:
+                if isinstance(lens, torch.Tensor):
+                    flat_feature_lens.extend(lens.flatten().tolist())
+                else:
+                    flat_feature_lens.append(int(lens))
             unpadded_audio_features = [
                 feat[:, :feature_len]
                 for feat, feature_len in zip(
                     audio_inputs["audio_features"],
-                    audio_inputs["audio_feature_lens"],
+                    flat_feature_lens,
                 )
             ]
             audio_inputs["audio_features"] = unpadded_audio_features