rebase: align vllm-omni with vLLM ac9126e58aa7

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>

tests/model_executor/models/omnivoice/test_processor_prompt_tokenization.py

@@ -42,10 +42,26 @@ def _make_processor(monkeypatch: pytest.MonkeyPatch) -> OmniVoiceMultiModalProce
     return processor
 
 
+class _FakeProcessorItem:
+    def __init__(self, processor_data: dict[str, object]) -> None:
+        self._processor_data = processor_data
+
+    def get_processor_data(self) -> dict[str, object]:
+        return dict(self._processor_data)
+
+    def get_passthrough_data(self) -> dict[str, object]:
+        return {}
+
+
 def _items(counts: dict[str, int], mm_data: dict[str, object]) -> SimpleNamespace:
+    class _Selected:
+        @staticmethod
+        def values():
+            return [_FakeProcessorItem(mm_data)] if mm_data else []
+
     return SimpleNamespace(
         get_all_counts=lambda: counts,
-        select=lambda _keys: mm_data,
+        select=lambda _keys: _Selected(),
     )
 
 
@@ -75,9 +91,8 @@ def test_apply_tokenizes_the_prompt_once(monkeypatch: pytest.MonkeyPatch) -> Non
 def test_media_path_encodes_audio_without_retokenizing(monkeypatch: pytest.MonkeyPatch) -> None:
     processor = _make_processor(monkeypatch)
     audio = np.zeros(2400, dtype=np.float32)
-    monkeypatch.setattr(processor, "_get_hf_mm_data", lambda _items: ({"audios": [audio]}, {}))
 
-    out = processor._apply_hf_processor_main(_items({"audio": 1}, {}), {"ref_text": "reference"})
+    out = processor._apply_hf_processor_main(_items({"audio": 1}, {"audios": [audio]}), {"ref_text": "reference"})
 
     assert tuple(out["ref_audio_tokens"].shape) == (8, 4)
     assert list(out["ref_audio_len"]) == [4]
@@ -89,7 +104,6 @@ def test_media_path_encodes_audio_without_retokenizing(monkeypatch: pytest.Monke
 def test_media_path_with_no_audio_touches_no_tokenizer(monkeypatch: pytest.MonkeyPatch) -> None:
     """Text-only requests and fully cached audio both reach this path with empty items."""
     processor = _make_processor(monkeypatch)
-    monkeypatch.setattr(processor, "_get_hf_mm_data", lambda _items: ({}, {}))
 
     out = processor._apply_hf_processor_main(_items({}, {}), {"ref_text": "reference"})
 
@@ -99,4 +113,6 @@ def test_media_path_with_no_audio_touches_no_tokenizer(monkeypatch: pytest.Monke
 
 
 def test_legacy_call_hf_processor_is_gone() -> None:
-    assert not hasattr(OmniVoiceMultiModalProcessor, "_call_hf_processor")
+    # Upstream reintroduced a 2-arg `_call_hf_processor` on the base class; the
+    # omni-specific 4-arg hook must not be defined on the omni class itself.
+    assert "_call_hf_processor" not in OmniVoiceMultiModalProcessor.__dict__

tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py

@@ -262,6 +262,7 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
         "audio": 1,
     }[modality]
     mm_data_items.__getitem__ = MagicMock(side_effect=lambda modality: [f"{modality}-0"])
+    mm_data_items.values = MagicMock(return_value=[])
 
     inputs = MagicMock()
     inputs.mm_data_items = mm_data_items
@@ -275,6 +276,7 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
     cache = MagicMock()
     # Video miss, audio hit under pair-aware keys — expansion must force audio.
     cache.is_cached.side_effect = lambda keys: [False] if keys[0].startswith("v") else [True]
+    inputs.cache = cache
 
     captured = {}
 
@@ -309,6 +311,8 @@ def test_cached_apply_hf_processor_expands_pair_miss_for_audio_hit_video_miss():
     fake_self._get_mm_fields_config = MagicMock(return_value={})
     fake_self._get_mm_prompt_updates = MagicMock(return_value={"video": [[object()]], "audio": [[object()]]})
     fake_self._merge_mm_kwargs = capture_merge
+    fake_self._postprocess_prompt = lambda prompt: prompt
+    inputs.prompt = [0, 1, 2]
 
     with patch(
         "vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.MultiModalKwargsItems.from_hf_inputs",
@@ -549,24 +553,6 @@ def test_qwen2_5_prompt_updates_validate_mask_against_request_video_count():
         )
 
 
-def test_qwen2_5_mm_only_dummy_counts_subtract_only_videos_using_audio():
-    captured_counts = {}
-    fake_self = SimpleNamespace(
-        dummy_inputs=SimpleNamespace(get_dummy_text=lambda counts: captured_counts.update(counts) or "dummy"),
-        _apply_hf_processor_text_mm=lambda **_kwargs: ([], {}, False),
-    )
-    mm_items = SimpleNamespace(get_all_counts=lambda: {"video": 2, "audio": 1})
-
-    Qwen2_5OmniThinkerMultiModalProcessor._apply_hf_processor_mm_only(
-        fake_self,
-        mm_items,
-        {"use_audio_in_video": [True, False]},
-        {},
-    )
-
-    assert captured_counts == {"video": 2, "audio": 0}
-
-
 def test_get_video_use_audio_in_video_falls_back_to_prompt_updates_for_cache_hits():
     fake_self = _fake_processor()
     update_with_audio = SimpleNamespace(content=SimpleNamespace(full=[VIDEO_TOKEN_ID, AUDIO_TOKEN_ID]))
@@ -661,7 +647,12 @@ def test_qwen3_processor_inherits_vllm_omni_per_video_helpers():
         Qwen3OmniMoeThinkerMultiModalProcessor,
         Qwen2_5OmniThinkerMultiModalProcessor,
     )
-    assert Qwen3OmniMoeThinkerMultiModalProcessor._get_hf_mm_data is UpstreamQwen3Processor._get_hf_mm_data
+    # Keep Qwen3's native HF preprocessing while inheriting Omni pair handling.
+    assert Qwen3OmniMoeThinkerMultiModalProcessor._get_hf_mm_inputs is UpstreamQwen3Processor._get_hf_mm_inputs
+    assert (
+        Qwen3OmniMoeThinkerMultiModalProcessor._get_audio_in_video_pairs
+        is Qwen2_5OmniThinkerMultiModalProcessor._get_audio_in_video_pairs
+    )
     mro = Qwen3OmniMoeThinkerMultiModalProcessor.__mro__
     assert mro.index(Qwen2_5OmniThinkerMultiModalProcessor) < mro.index(UpstreamQwen3Processor)
 

tests/model_executor/models/test_omni_processing.py

@@ -79,31 +79,16 @@ def get_text_token_prompts(
     """Return ``(text_prompt, token_prompt)`` for the given multimodal data."""
     dummy_inputs = processor.dummy_inputs
     tokenizer: TokenizerLike = processor.info.get_tokenizer()
-    model_config = processor.info.ctx.model_config
 
     parsed_data = processor.info.parse_mm_data(mm_data)
     mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
 
-    inputs = dummy_inputs.get_dummy_processor_inputs(
-        model_config.max_model_len,
-        mm_counts,
-        mm_options={},
+    text_prompt = dummy_inputs.get_dummy_text(mm_counts)
+    token_prompt = tokenizer.encode(
+        text_prompt,
+        **processor.info.get_default_tok_params().get_encode_kwargs(),
     )
 
-    text_prompt: str | None
-    token_prompt: list[int]
-    if isinstance(inputs.prompt, list):
-        text_prompt = None
-        token_prompt = inputs.prompt
-    elif isinstance(inputs.prompt, str):
-        text_prompt = inputs.prompt
-        token_prompt = tokenizer.encode(
-            text_prompt,
-            **processor.info.get_default_tok_params().get_encode_kwargs(),
-        )
-    else:
-        raise TypeError(type(inputs.prompt))
-
     return text_prompt, token_prompt
 
 
@@ -199,8 +184,8 @@ def _test_processing_correctness(
         modality: _to_dummy_options(modality, count) for modality, count in limit_mm_per_prompt_ints.items()
     }
 
-    baseline_processor = factories.build_processor(ctx, cache=None)
-    cached_processor = factories.build_processor(ctx, cache=cache)
+    baseline_processor = factories.build_processor(ctx)
+    cached_processor = factories.build_processor(ctx)
 
     rng = np.random.RandomState(0)
 
@@ -238,6 +223,7 @@ def _test_processing_correctness(
             baseline_processor,
             cached_processor,
             batch_idx,
+            cache,
         )
 
 
@@ -247,6 +233,7 @@ def _test_processing_correctness_one(
     baseline_processor: BaseMultiModalProcessor,
     cached_processor: BaseMultiModalProcessor,
     batch_idx: int,
+    cache: MultiModalProcessorOnlyCache,
 ):
     model_type = model_config.hf_config.model_type
 
@@ -258,12 +245,14 @@ def _test_processing_correctness_one(
         token_prompt,
         mm_items=mm_items,
         hf_processor_mm_kwargs={},
+        cache=None,
     )
 
     cached_tokenized_result = cached_processor(
         token_prompt,
         mm_items=mm_items,
         hf_processor_mm_kwargs={},
+        cache=cache,
     )
 
     _assert_inputs_equal(
@@ -278,11 +267,13 @@ def _test_processing_correctness_one(
             text_prompt,
             mm_items=mm_items,
             hf_processor_mm_kwargs={},
+            cache=None,
         )
         cached_text_result = cached_processor(
             text_prompt,
             mm_items=mm_items,
             hf_processor_mm_kwargs={},
+            cache=cache,
         )
 
         _assert_inputs_equal(