refactor: reduce copied vLLM 0.29 compatibility code
Reuse upstream weight mapping and multimodal processing, consolidate Omni processor bridges, and trim unused Hunyuan internals. Route no-media kwargs through the active renderer and preserve GLM generation prompt formatting. Validation: 453 CPU tests passed, 7 skipped; processor imports, Ruff, and diff checks passed. GPU and end-to-end CI not rerun. Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
eef8e2bb94
commit
efda9369a8
37 files changed
+448
-1506
No files matched your search
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Multimodal data plumbing for Omni's custom, non-HF processors."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TypeVar
|
||||
|
||||
from transformers import BatchFeature
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, BaseProcessingInfo
|
||||
|
||||
_I = TypeVar("_I", bound=BaseProcessingInfo)
|
||||
|
||||
|
||||
class OmniMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
"""Share data/passthrough handling, leaving model processing in its owner.
|
||||
|
||||
These processors implement ``_call_hf_processor`` locally, sometimes
|
||||
without any HF processor object. Prompt tokens are prepared separately
|
||||
in ``apply``; the upstream base still owns caching and prompt updates.
|
||||
"""
|
||||
|
||||
_OMNI_PROMPT_TEXT_KEY = "_vllm_omni_original_prompt_text"
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
kwargs = dict(hf_processor_mm_kwargs)
|
||||
prompt = kwargs.pop(self._OMNI_PROMPT_TEXT_KEY, None)
|
||||
if prompt is None:
|
||||
prompt = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
valid_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
|
||||
mm_data, passthrough = self._get_hf_mm_data(valid_items)
|
||||
result = self._call_hf_processor(str(prompt), dict(mm_data), kwargs, {})
|
||||
result.update(passthrough)
|
||||
return result
|
||||
Reference in new issue
Block a user