refactor(processors): drop _call_hf_processor overrides upstream no longer calls
vLLM 0.29 removed BaseMultiModalProcessor._call_hf_processor; the pipeline
now reaches the HF processor through _apply_hf_processor_main and the
_preprocess_hf_mm_data / _postprocess_hf_mm_data hooks. These three
overrides still called super()._call_hf_processor, which would raise
AttributeError if anything reached them:
- bagel: _apply_hf_processor_main already performs the image / img2img
split via self.info.ctx.call_hf_processor, so the override was unreachable.
- mimo_audio_llm: the audios -> audio rename and the sampling_rate kwarg
already live in _preprocess_hf_mm_data.
- step_audio2: the audios -> audio normalisation and the empty
audio_mels / audio_lens defaults already live in _preprocess_hf_mm_data
and _postprocess_hf_mm_data.
Caller audit: the only remaining _call_hf_processor call sites are the live
ones in vllm_omni/inputs/mm_processor.py, omnivoice.py and the GLM-TTS test.
Validation: ruff and pre-commit on the three files; the three modules
import cleanly; CPU tests in tests/inputs, tests/model_executor/models/
{mimo_audio,step_audio2} and test_upstream_prompt_update_api.py pass.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MpXe2LeRrUQGjZfKFaocMZ
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouamandClaude Fable 5.1 committed
1 parent
188d021023
commit
1439847f33
3 files changed
-123
No files matched your search
@@ -256,61 +256,6 @@ class OmniBagelMultiModalProcessor(BaseMultiModalProcessor[OmniBagelProcessingIn
|
||||
"pixel_values_img2img": MultiModalFieldConfig.batched("img2img"),
|
||||
}
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
has_image = "images" in mm_data
|
||||
has_img2img = "pixel_values_img2img" in mm_data
|
||||
|
||||
if has_img2img and self.IMG2IMG_PLACEHOLDER not in prompt:
|
||||
prompt = f"{self.IMG2IMG_PLACEHOLDER}{prompt}"
|
||||
|
||||
if has_image and has_img2img:
|
||||
outputs = BatchFeature()
|
||||
|
||||
img_data = dict(mm_data)
|
||||
if "pixel_values_img2img" in img_data:
|
||||
del img_data["pixel_values_img2img"]
|
||||
kwargs_img = dict(mm_kwargs)
|
||||
kwargs_img["is_img2img"] = False
|
||||
out_img = super()._call_hf_processor(prompt, img_data, kwargs_img, tok_kwargs)
|
||||
if "pixel_values" in out_img:
|
||||
outputs["pixel_values"] = out_img["pixel_values"]
|
||||
for k, v in out_img.items():
|
||||
if k != "pixel_values":
|
||||
outputs[k] = v
|
||||
|
||||
img2img_data = dict(mm_data)
|
||||
if "images" in img2img_data:
|
||||
del img2img_data["images"]
|
||||
img2img_data["images"] = img2img_data.pop("pixel_values_img2img")
|
||||
kwargs_img2img = self._mm_kwargs_for_bagel_img2img_hf(mm_kwargs)
|
||||
kwargs_img2img["is_img2img"] = True
|
||||
out_img2img = super()._call_hf_processor(prompt, img2img_data, kwargs_img2img, tok_kwargs)
|
||||
if "pixel_values" in out_img2img:
|
||||
outputs["pixel_values_img2img"] = out_img2img["pixel_values"]
|
||||
for k, v in out_img2img.items():
|
||||
if k not in outputs:
|
||||
outputs[k] = v
|
||||
|
||||
return outputs
|
||||
|
||||
elif has_img2img:
|
||||
mm_data = dict(mm_data)
|
||||
mm_data["images"] = mm_data.pop("pixel_values_img2img")
|
||||
mm_kwargs = self._mm_kwargs_for_bagel_img2img_hf(mm_kwargs)
|
||||
mm_kwargs["is_img2img"] = True
|
||||
outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
if "pixel_values" in outputs:
|
||||
outputs["pixel_values_img2img"] = outputs.pop("pixel_values")
|
||||
return outputs
|
||||
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
@@ -390,36 +390,6 @@ class MimoAudioDummyInputsBuilder(BaseDummyInputsBuilder[MimoAudioProcessingInfo
|
||||
|
||||
|
||||
class MimoAudioMultiModalProcessor(BaseMultiModalProcessor[MimoAudioProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, Any],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
mm_data["audio"] = audios
|
||||
|
||||
# Text-only input not supported in composite processor
|
||||
if not mm_data.get("audio", []):
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
sampling_rate=feature_extractor.sampling_rate,
|
||||
)
|
||||
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -561,44 +561,6 @@ class StepAudio2MultiModalProcessor(BaseMultiModalProcessor[StepAudio2Processing
|
||||
)
|
||||
]
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object] | None = None,
|
||||
tok_kwargs: Mapping[str, object] | None = None,
|
||||
**_: object,
|
||||
):
|
||||
"""Call HF processor and post-process outputs"""
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
mm_kwargs = mm_kwargs or {}
|
||||
tok_kwargs = tok_kwargs or {}
|
||||
|
||||
# CRITICAL: Ensure audios is ALWAYS a list to prevent string iteration
|
||||
if audios:
|
||||
if isinstance(audios, str):
|
||||
mm_data["audio"] = [audios]
|
||||
elif isinstance(audios, (list, tuple)):
|
||||
mm_data["audio"] = audios
|
||||
else:
|
||||
mm_data["audio"] = [audios]
|
||||
|
||||
hf_inputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
if "audio_mels" not in hf_inputs:
|
||||
hf_inputs["audio_mels"] = torch.empty((0, 128, 0))
|
||||
if "audio_lens" not in hf_inputs:
|
||||
hf_inputs["audio_lens"] = torch.tensor([], dtype=torch.int32)
|
||||
|
||||
return hf_inputs
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
Reference in new issue
Block a user