refactor: reduce copied vLLM 0.29 compatibility code
Reuse upstream weight mapping and multimodal processing, consolidate Omni processor bridges, and trim unused Hunyuan internals. Route no-media kwargs through the active renderer and preserve GLM generation prompt formatting. Validation: 453 CPU tests passed, 7 skipped; processor imports, Ruff, and diff checks passed. GPU and end-to-end CI not rerun. Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
eef8e2bb94
commit
efda9369a8
37 files changed
+448
-1506
No files matched your search
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("counts", [{}, {"audio": 1, "image": 0}])
|
||||
@pytest.mark.parametrize("original_text", [None, "", "voice cloning text"])
|
||||
def test_custom_processor_bridge_preserves_prompt_kwargs_and_passthrough(counts, original_text):
|
||||
class Processor(OmniMultiModalProcessor):
|
||||
def _get_mm_fields_config(self, *args):
|
||||
return {}
|
||||
|
||||
def _get_prompt_updates(self, *args):
|
||||
return []
|
||||
|
||||
processor = object.__new__(Processor)
|
||||
processor.dummy_inputs = SimpleNamespace(get_dummy_text=Mock(return_value="dummy"))
|
||||
processor._get_hf_mm_data = Mock(return_value=({"audios": ["wave"]}, {"embedding": "preserve"}))
|
||||
processor._call_hf_processor = Mock(return_value=BatchFeature({"features": "processed"}))
|
||||
items = Mock()
|
||||
items.get_all_counts.return_value = counts
|
||||
kwargs = {"sampling_rate": 16000}
|
||||
if original_text is not None:
|
||||
kwargs[processor._OMNI_PROMPT_TEXT_KEY] = original_text
|
||||
before = dict(kwargs)
|
||||
result = processor._apply_hf_processor_main(items, kwargs)
|
||||
items.select.assert_called_once_with({k for k, v in counts.items() if v > 0})
|
||||
processor._call_hf_processor.assert_called_once_with(
|
||||
"dummy" if original_text is None else original_text,
|
||||
{"audios": ["wave"]},
|
||||
{"sampling_rate": 16000},
|
||||
{},
|
||||
)
|
||||
assert dict(result) == {"features": "processed", "embedding": "preserve"}
|
||||
assert kwargs == before
|
||||
@@ -1,35 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Unit tests for OmniInputPreprocessor._process_text routing."""
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Exercise Omni routing through the active upstream rendering pipeline."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
import torch
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.renderers.params import TokenizeParams
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
from vllm_omni.inputs.preprocess import OmniRenderer
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class TestProcessTextMmProcessorKwargsRouting:
|
||||
"""Presence-based routing: an explicit empty ``mm_processor_kwargs`` dict
|
||||
on the prompt still routes through ``_process_multimodal``. Required for
|
||||
AR-based image generation (e.g. GLM-Image t2i) where the HF processor
|
||||
supplies the image-generation scaffold from its own defaults and no extra
|
||||
kwargs are needed from the caller.
|
||||
"""
|
||||
class _Renderer(BaseRenderer):
|
||||
def __init__(self):
|
||||
self.model_config = SimpleNamespace(is_encoder_decoder=False, enable_prompt_embeds=True)
|
||||
self.tokenizer = None
|
||||
self.default_cmpl_tok_params = TokenizeParams(max_total_tokens=128)
|
||||
self._tokenize_prompt = lambda prompt, params: {**prompt, "prompt_token_ids": [1, 2, 3]}
|
||||
self._tokenize_prompt_async = AsyncMock(side_effect=self._tokenize_prompt)
|
||||
self._process_multimodal = Mock(return_value=tokens_input([1, 2, 3, 99]))
|
||||
self._process_multimodal_async = AsyncMock(side_effect=self._process_multimodal)
|
||||
|
||||
@pytest.fixture
|
||||
def preprocessor(self, mocker: MockerFixture):
|
||||
from vllm_omni.inputs.preprocess import OmniInputPreprocessor
|
||||
def render_messages(self, messages, params):
|
||||
return messages, {"prompt": "hello"}
|
||||
|
||||
instance = object.__new__(OmniInputPreprocessor)
|
||||
instance._process_multimodal = mocker.MagicMock(return_value={})
|
||||
instance._tokenize_prompt = mocker.MagicMock(return_value=[1, 2, 3])
|
||||
return instance
|
||||
|
||||
def test_empty_mm_processor_kwargs_routes_to_multimodal(self, preprocessor):
|
||||
preprocessor._process_text({"prompt": "hello", "mm_processor_kwargs": {}})
|
||||
assert preprocessor._process_multimodal.called
|
||||
assert not preprocessor._tokenize_prompt.called
|
||||
@pytest.fixture
|
||||
def renderer():
|
||||
return OmniRenderer(_Renderer())
|
||||
|
||||
def test_missing_mm_processor_kwargs_routes_to_tokenize(self, preprocessor):
|
||||
preprocessor._process_text({"prompt": "hello"})
|
||||
assert preprocessor._tokenize_prompt.called
|
||||
assert not preprocessor._process_multimodal.called
|
||||
|
||||
@pytest.mark.parametrize("kwargs", [{}, {"target_h": 512, "target_w": 768}])
|
||||
@pytest.mark.parametrize("tokenized", [False, True])
|
||||
def test_process_inputs_routes_no_media_processor_kwargs(renderer, kwargs, tokenized):
|
||||
# Keep real process_inputs/render_cmpl; stub unrelated config validation.
|
||||
processor = object.__new__(InputProcessor)
|
||||
processor.renderer = renderer
|
||||
processor.model_config = renderer.model_config
|
||||
processor.vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=1,
|
||||
data_parallel_size_local=1,
|
||||
local_engines_only=False,
|
||||
)
|
||||
)
|
||||
processor._validate_params = Mock()
|
||||
processor._validate_lora = Mock()
|
||||
processor._validate_model_inputs = Mock()
|
||||
prompt = {"prompt_token_ids": [1, 2, 3]} if tokenized else {"prompt": "hello"}
|
||||
prompt.update(mm_processor_kwargs=kwargs, cache_salt="salt")
|
||||
request = processor.process_inputs("image-request", prompt, PoolingParams(), ("embed",))
|
||||
assert request.prompt_token_ids == [1, 2, 3, 99]
|
||||
assert request.cache_salt == "salt"
|
||||
renderer._renderer._process_multimodal.assert_called_once_with(
|
||||
[1, 2, 3],
|
||||
{},
|
||||
mm_processor_kwargs=kwargs,
|
||||
mm_uuids=None,
|
||||
skip_mm_cache=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_mode", [False, True])
|
||||
@pytest.mark.parametrize("kind", ["text", "tokens", "media", "kwargs", "embeds"])
|
||||
def test_renderer_preserves_routes_extras_and_cache_policy(renderer, async_mode, kind):
|
||||
prompt = {"prompt": "hello", "additional_information": {"speaker": 1}, "model_intermediate_buffer": {"x": 2}}
|
||||
if kind == "tokens":
|
||||
prompt["prompt_token_ids"] = [1, 2, 3]
|
||||
elif kind == "media":
|
||||
prompt["multi_modal_data"] = {"image": "image"}
|
||||
elif kind == "kwargs":
|
||||
prompt["mm_processor_kwargs"] = {}
|
||||
elif kind == "embeds":
|
||||
prompt["prompt_embeds"] = torch.ones(3, 4)
|
||||
if async_mode:
|
||||
(result,) = asyncio.run(renderer.render_cmpl_async([prompt], skip_mm_cache=True))
|
||||
else:
|
||||
(result,) = renderer.render_cmpl([prompt], skip_mm_cache=True)
|
||||
assert result["additional_information"] == {"speaker": 1}
|
||||
assert result["model_intermediate_buffer"] == {"x": 2}
|
||||
assert result["prompt"] == "hello"
|
||||
if kind in ("media", "kwargs"):
|
||||
assert result["prompt_token_ids"] == [1, 2, 3, 99]
|
||||
assert renderer._renderer._process_multimodal.call_args.kwargs["skip_mm_cache"] is True
|
||||
else:
|
||||
renderer._renderer._process_multimodal.assert_not_called()
|
||||
if kind == "embeds":
|
||||
torch.testing.assert_close(result["prompt_embeds"], prompt["prompt_embeds"])
|
||||
else:
|
||||
assert result["prompt_token_ids"] == [1, 2, 3]
|
||||
Reference in new issue
Block a user