From dd1389173b4e2893d21cf742979c89ab0255a5d5 Mon Sep 17 00:00:00 2001 From: Chen-Yo Sun Date: Mon, 13 Apr 2026 15:37:45 -0700 Subject: [PATCH] [Voxtral TTS] Fix Voxtral TTS input with text and ref_audio (#2750) Signed-off-by: Chen-Yo Sun --- .../voxtral_tts_audio_generation.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py b/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py index 4041a53e5..cd67e4f07 100644 --- a/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py +++ b/vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py @@ -864,6 +864,29 @@ class VoxtralTTSMultiModalProcessor(BaseMultiModalProcessor[VoxtralTTSProcessing ), ] + def _apply_hf_processor_mm_only( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + tokenization_kwargs: Mapping[str, object], + ) -> BatchFeature: + """ + Apply the HF processor on the multi-modal data only. + + Issue: Voxtral TTS use Mistral Tokenizer with custom audio encoder. It doesn't + inherit Transformers ProcessorMixin and can't use call_hf_processor_mm_only. + + Solution: Override this method to call _apply_hf_processor_text_mm directly. + """ + mm_counts = mm_items.get_all_counts() + _, mm_processed_data, _ = self._apply_hf_processor_text_mm( + prompt_text=self.dummy_inputs.get_dummy_text(mm_counts), + mm_items=mm_items, + hf_processor_mm_kwargs=hf_processor_mm_kwargs, + tokenization_kwargs=tokenization_kwargs, + ) + return mm_processed_data + def _cached_apply_hf_processor( self, inputs: ProcessorInputs,