[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>

tests/inputs/test_mm_processor.py

@@ -1,13 +1,17 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
+from importlib import import_module
 from types import SimpleNamespace
 from unittest.mock import Mock
 
 import pytest
 from transformers import BatchFeature
+from vllm.config.multimodal import MultiModalDummyOptions
+from vllm.multimodal.parse import AudioProcessorItems, ImageProcessorItems, MultiModalDataItems
+from vllm.multimodal.processing import BaseMultiModalProcessor
 
-from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
+from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
@@ -24,10 +28,14 @@ def test_custom_processor_bridge_preserves_prompt_kwargs_and_passthrough(counts,
 
     processor = object.__new__(Processor)
     processor.dummy_inputs = SimpleNamespace(get_dummy_text=Mock(return_value="dummy"))
-    processor._get_hf_mm_data = Mock(return_value=({"audios": ["wave"]}, {"embedding": "preserve"}))
     processor._call_hf_processor = Mock(return_value=BatchFeature({"features": "processed"}))
-    items = Mock()
-    items.get_all_counts.return_value = counts
+
+    class AudioItems(AudioProcessorItems):
+        def get_passthrough_data(self):
+            return {"embedding": "preserve"}
+
+    items = MultiModalDataItems({"audio": AudioItems(["wave"]), "image": ImageProcessorItems([])} if counts else {})
+    items.select = Mock(wraps=items.select)
     kwargs = {"sampling_rate": 16000}
     if original_text is not None:
         kwargs[processor._OMNI_PROMPT_TEXT_KEY] = original_text
@@ -36,9 +44,83 @@ def test_custom_processor_bridge_preserves_prompt_kwargs_and_passthrough(counts,
     items.select.assert_called_once_with({k for k, v in counts.items() if v > 0})
     processor._call_hf_processor.assert_called_once_with(
         "dummy" if original_text is None else original_text,
-        {"audios": ["wave"]},
+        {"audios": ["wave"]} if counts else {},
         {"sampling_rate": 16000},
         {},
     )
-    assert dict(result) == {"features": "processed", "embedding": "preserve"}
+    assert dict(result) == ({"features": "processed", "embedding": "preserve"} if counts else {"features": "processed"})
     assert kwargs == before
+
+
+@pytest.mark.parametrize(
+    "module_name,builder_name,processor_name,expected_kwargs",
+    [
+        (
+            "cosyvoice3",
+            "CosyVoice3DummyInputsBuilder",
+            "CosyVoice3MultiModalProcessor",
+            {"prompt_text": "Testing my voices. Why should I not?"},
+        ),
+        (
+            "glm_tts",
+            "GLMTTSDummyInputsBuilder",
+            "GLMTTSMultiModalProcessor",
+            {"prompt_text": "This is the reference voice."},
+        ),
+        (
+            "omnivoice",
+            "OmniVoiceDummyInputsBuilder",
+            "OmniVoiceMultiModalProcessor",
+            {"ref_text": "Testing voice cloning."},
+        ),
+        ("mimo_audio", "MiMoAudioLLMDummyInputsBuilder", "MiMoAudioLLMMultiModalProcessor", {}),
+    ],
+)
+def test_dummy_budget_preserves_model_conditioning(module_name, builder_name, processor_name, expected_kwargs):
+    """The v0.31 budget path must reach each model's custom builder hook."""
+    module = import_module(f"vllm_omni.model_executor.models.{module_name}.{module_name}")
+    builder = object.__new__(getattr(module, builder_name))
+    assert isinstance(builder, OmniDummyInputsBuilder)
+    items = MultiModalDataItems({})
+    builder.info = SimpleNamespace(ctx=SimpleNamespace(tokenizer=None), parse_mm_data=Mock(return_value=items))
+    builder.get_dummy_mm_data = Mock(return_value={})
+    processor = object.__new__(getattr(module, processor_name))
+    processor.dummy_inputs = builder
+
+    inputs = processor.get_dummy_inputs(128, {"audio": 1}, MultiModalDummyOptions())
+
+    assert inputs.mm_data_items is items
+    assert inputs.hf_processor_mm_kwargs == expected_kwargs
+    assert inputs.prompt == ("<|empty|>" if module_name == "mimo_audio" else [])
+
+
+def test_dummy_budget_keeps_upstream_path_for_standard_builders(monkeypatch):
+    class Processor(OmniMultiModalProcessor):
+        def _get_mm_fields_config(self, *args):
+            return {}
+
+        def _get_prompt_updates(self, *args):
+            return []
+
+    processor = object.__new__(Processor)
+    processor.dummy_inputs = SimpleNamespace()
+    expected = object()
+    upstream = Mock(return_value=expected)
+    monkeypatch.setattr(BaseMultiModalProcessor, "get_dummy_inputs", upstream)
+    options = MultiModalDummyOptions()
+
+    assert processor.get_dummy_inputs(128, {"audio": 1}, options) is expected
+    upstream.assert_called_once_with(128, {"audio": 1}, options)
+
+
+def test_voxtral_dummy_budget_uses_audio_token_prompt():
+    from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import VoxtralTTSMultiModalProcessor
+
+    processor = object.__new__(VoxtralTTSMultiModalProcessor)
+    expected = SimpleNamespace(prompt=[1, 25, 24, 35])
+    builder = SimpleNamespace(get_dummy_processor_inputs=Mock(return_value=expected))
+    processor.dummy_inputs = builder
+    options = MultiModalDummyOptions()
+
+    assert processor.get_dummy_inputs(128, {"audio": 1}, options) is expected
+    builder.get_dummy_processor_inputs.assert_called_once_with(128, {"audio": 1}, options)

tests/inputs/test_preprocess.py

@@ -8,7 +8,9 @@ from unittest.mock import AsyncMock, Mock
 
 import pytest
 import torch
-from vllm.inputs import tokens_input
+from vllm.exceptions import VLLMValidationError
+from vllm.inputs import mm_input, tokens_input
+from vllm.multimodal.inputs import PlaceholderRange
 from vllm.pooling_params import PoolingParams
 from vllm.renderers import BaseRenderer
 from vllm.renderers.params import TokenizeParams
@@ -115,6 +117,35 @@ def test_omni_renderer_is_a_real_subclass_created_once():
     assert not hasattr(instance, "_renderer")
 
 
+@pytest.mark.parametrize("async_mode", [False, True])
+@pytest.mark.parametrize("no_media_kwargs", [False, True])
+@pytest.mark.parametrize("limit", [2, 4])
+def test_renderer_applies_expanded_prompt_bound_without_splitting_media(renderer, async_mode, no_media_kwargs, limit):
+    renderer._process_multimodal.return_value = mm_input(
+        list(range(6)), {}, {"image": ["hash"]}, {"image": [PlaceholderRange(offset=1, length=2)]}
+    )
+    prompt = {"prompt_token_ids": [1, 2, 3], "additional_information": {"speaker": 1}}
+    if no_media_kwargs:
+        prompt["mm_processor_kwargs"] = {"target_h": 512}
+    else:
+        prompt["multi_modal_data"] = {"image": "image"}
+    params = TokenizeParams(max_total_tokens=128, truncate_prompt_tokens=limit, truncation_side="right")
+
+    def render():
+        if async_mode:
+            return asyncio.run(renderer.render_cmpl_async([prompt], params))
+        return renderer.render_cmpl([prompt], params)
+
+    if limit == 2:
+        with pytest.raises(VLLMValidationError, match="would split or drop"):
+            render()
+    else:
+        (result,) = render()
+        assert result["prompt_token_ids"] == [0, 1, 2, 3]
+        assert result["mm_placeholders"]["image"] == [PlaceholderRange(offset=1, length=2)]
+        assert result["additional_information"] == {"speaker": 1}
+
+
 def test_concrete_renderer_overrides_are_not_suppressed():
     """The old proxy bound BaseRenderer.render_cmpl itself and hid subclass overrides."""