[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
Zhou Taichang authored and GitHub committed 2026-10-05 14:11:01 +08:00
1 parent 1f47fa2f29
commit 0b0d2d7009
141 files changed
+2885 -721

No files matched your search

+11 -5
View File
@@ -259,19 +259,25 @@ class OmniBagelMultiModalProcessor(BaseMultiModalProcessor[OmniBagelProcessingIn
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
mm_data: dict[str, object] = {}
passthrough_data: dict[str, object] = {}
for items in valid_mm_items.values():
if not items:
continue
mm_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
processor = self.info.get_hf_processor(**hf_kwargs)
has_image = "images" in mm_data
has_img2img = "pixel_values_img2img" in mm_data
processed_data = BatchFeature()
if has_image:
image_kwargs = {**hf_processor_mm_kwargs, "is_img2img": False}
image_kwargs = {**hf_kwargs, "is_img2img": False}
image_outputs = self.info.ctx.call_hf_processor(
processor,
{"text": prompt_text, "images": mm_data["images"]},
@@ -280,7 +286,7 @@ class OmniBagelMultiModalProcessor(BaseMultiModalProcessor[OmniBagelProcessingIn
processed_data.update(image_outputs)
if has_img2img:
img2img_kwargs = self._mm_kwargs_for_bagel_img2img_hf(hf_processor_mm_kwargs)
img2img_kwargs = self._mm_kwargs_for_bagel_img2img_hf(hf_kwargs)
img2img_kwargs["is_img2img"] = True
img2img_outputs = self.info.ctx.call_hf_processor(
processor,
@@ -89,14 +89,12 @@ def _cfg_extra(request: Any, key: str) -> Any:
def _wait_queues(scheduler: Any) -> list[Any]:
# Divergence from the official (vLLM 0.20-era) file: vLLM 0.24 parks
# structurally not-ready requests in a second ``skipped_waiting`` queue;
# pair-hold must cover both queues or a lone member parked there would
# be admitted without its partner.
# Pair holding must cover requests that retain KV blocks as well as
# fresh requests, or a lone member can be admitted without its partner.
queues = [scheduler.waiting]
skipped = getattr(scheduler, "skipped_waiting", None)
if skipped is not None:
queues.append(skipped)
kv_holding = getattr(scheduler, "kv_holding_waiting", None)
if kv_holding is not None:
queues.append(kv_holding)
return queues
@@ -108,7 +106,7 @@ def _pair_complete(scheduler: Any, request_id: str, blocked_ids: set[str] | None
if len(roles) != 2 or not all(rid in scheduler.requests for rid in roles.values()):
return False
if blocked_ids:
# A partner parked in skipped_waiting (e.g. waiting for remote KVs)
# A deferred partner (e.g. waiting for remote KVs)
# may fail promotion this step while this member gets scheduled from
# waiting; hold this member until the partner is schedulable.
for rid in roles.values():
@@ -149,14 +147,14 @@ def _hold_incomplete_pairs(scheduler: Any) -> list[tuple[Any, Any]]:
hold_counts: dict[str, int] = getattr(scheduler, "_cfg_hold_counts", None) or {}
scheduler._cfg_hold_counts = hold_counts
skipped = getattr(scheduler, "skipped_waiting", None)
blocked_ids = {req.request_id for req in list(skipped)} if skipped is not None else set()
kv_holding = getattr(scheduler, "kv_holding_waiting", None)
blocked_ids = {req.request_id for req in getattr(scheduler, "deferred_waiting", ())}
held: list[tuple[Any, Any]] = []
for queue in _wait_queues(scheduler):
# A member is only "blocked" for its partner's sake when it sits in
# the OTHER queue; members of the queue being scanned move together.
partner_blockers = blocked_ids if queue is not skipped else set()
partner_blockers = blocked_ids if queue is not kv_holding else set()
held_here = []
for request in list(queue):
if _pair_complete(scheduler, request.request_id, partner_blockers):
@@ -181,8 +179,12 @@ def _release_held(held: list[tuple[Any, Any]]) -> None:
def _reorder_waiting_for_cfg(scheduler: Any) -> None:
"""Move CFG pair partners adjacent in the FCFS waiting queue."""
waiting = scheduler.waiting
"""Move CFG pair partners adjacent in each FCFS waiting queue."""
for waiting in _wait_queues(scheduler):
_reorder_cfg_queue(scheduler, waiting)
def _reorder_cfg_queue(scheduler: Any, waiting: Any) -> None:
# Only the FCFS queue (a deque) has caller-controlled ordering; priority
# queues order themselves and pair sync then relies on hold + equalize.
if not isinstance(waiting, deque) or len(waiting) < 2:
@@ -28,7 +28,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseProcessingInfo,
ProcessorInputs,
PromptIndexTargets,
@@ -42,7 +41,7 @@ from vllm.v1.sample.ops.topk_topp_sampler import random_sample
from vllm.v1.sample.sampler import Sampler
from vllm_omni.data_entry_keys import EmbeddingsStruct, OmniPayloadStruct, to_dict, to_struct
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
from vllm_omni.model_executor.models.cosyvoice3.ras_sampler import MAX_FUSED_TOP_K, fused_ras_sample
from vllm_omni.model_executor.models.cosyvoice3.runtime import (
cosyvoice3_batch_flow_debug,
@@ -696,7 +695,7 @@ class CosyVoice3MultiModalProcessor(OmniMultiModalProcessor[CosyVoice3MultiModal
]
class CosyVoice3DummyInputsBuilder(BaseDummyInputsBuilder[CosyVoice3MultiModalProcessingInfo]):
class CosyVoice3DummyInputsBuilder(OmniDummyInputsBuilder[CosyVoice3MultiModalProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return "Hello, this is a test of the CosyVoice3 system capability."
@@ -459,16 +459,22 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Process source images under vLLM's token-only prompt contract."""
mm_counts = mm_items.get_all_counts()
num_images = mm_counts.get("image", 0)
if num_images == 0:
return BatchFeature({"mrope_image_grid_thw": self._build_generation_grids(hf_processor_mm_kwargs)})
return BatchFeature({"mrope_image_grid_thw": self._build_generation_grids(hf_kwargs)})
valid_mm_items = mm_items.select({key for key, count in mm_counts.items() if count > 0})
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
processor_data: dict[str, object] = {}
passthrough_data: dict[str, object] = {}
for items in valid_mm_items.values():
if not items:
continue
processor_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
images = processor_data.get("images")
if not images:
return BatchFeature(dict(passthrough_data))
@@ -483,7 +489,7 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
source_grid_thw = image_grid_thw[:num_images]
image_inputs["image_grid_thw"] = source_grid_thw
target_grid = self._build_generation_grids(hf_processor_mm_kwargs)[:1].to(dtype=source_grid_thw.dtype)
target_grid = self._build_generation_grids(hf_kwargs)[:1].to(dtype=source_grid_thw.dtype)
image_inputs["mrope_image_grid_thw"] = torch.cat(
[source_grid_thw, target_grid],
dim=0,
@@ -38,7 +38,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseProcessingInfo,
ProcessorInputs,
PromptIndexTargets,
@@ -50,7 +49,7 @@ from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
from vllm_omni.model_executor.models.common.nucleus_ras_sampling import ras_sample_one as _ras_sample_one
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.platforms import current_omni_platform
@@ -674,7 +673,7 @@ class GLMTTSMultiModalProcessor(OmniMultiModalProcessor[GLMTTSMultiModalProcessi
]
class GLMTTSDummyInputsBuilder(BaseDummyInputsBuilder[GLMTTSMultiModalProcessingInfo]):
class GLMTTSDummyInputsBuilder(OmniDummyInputsBuilder[GLMTTSMultiModalProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return "This is a test of the GLM-TTS voice cloning system."
@@ -29,7 +29,6 @@ from vllm.multimodal.inputs import (
)
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseProcessingInfo,
ProcessorInputs,
PromptReplacement,
@@ -44,7 +43,7 @@ from vllm.utils.platform_utils import is_pin_memory_available
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.models.mimo_audio.config_mimo_audio import (
NO_INTERLEAVE_NEXT_TOKEN_ID,
@@ -224,7 +223,7 @@ class MiMoAudioLLMProcessingInfo(
return {"audio": 1}
class MiMoAudioLLMDummyInputsBuilder(BaseDummyInputsBuilder[MiMoAudioLLMProcessingInfo]):
class MiMoAudioLLMDummyInputsBuilder(OmniDummyInputsBuilder[MiMoAudioLLMProcessingInfo]):
_processor_inputs_cache: LRUCache = LRUCache(capacity=1024)
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
@@ -372,7 +371,7 @@ class MiMoAudioLLMMultiModalProcessor(OmniMultiModalProcessor[MiMoAudioLLMProces
# Text-only input not supported in composite processor
if not mm_data.get("audio", []):
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
prompt_ids = self._postprocess_prompt(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
@@ -396,25 +396,21 @@ class MimoAudioDummyInputsBuilder(BaseDummyInputsBuilder[MimoAudioProcessingInfo
class MimoAudioMultiModalProcessor(BaseMultiModalProcessor[MimoAudioProcessingInfo]):
def _preprocess_hf_mm_data(
def _get_hf_mm_inputs(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data, hf_processor_mm_kwargs = super()._preprocess_hf_mm_data(
mm_data,
hf_processor_mm_kwargs,
)
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
mm_data["audio"] = audios
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
hf_processor_mm_kwargs = {
**hf_processor_mm_kwargs,
"sampling_rate": feature_extractor.sampling_rate,
}
return mm_data, hf_processor_mm_kwargs
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
):
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
if hf_inputs.hf_data.get("audio"):
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
hf_inputs = hf_inputs._replace(
hf_kwargs={
**hf_kwargs,
"sampling_rate": feature_extractor.sampling_rate,
},
)
return hf_inputs
def _get_mm_fields_config(
self,
@@ -516,7 +516,7 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Call Ming's modality processors without expanding prompt text.
@@ -525,7 +525,13 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
placeholder tokens remain under vLLM's prompt-update control.
"""
valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
mm_data: dict[str, object] = {}
passthrough_data: dict[str, object] = {}
for items in valid_mm_items.values():
if not items:
continue
mm_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
hf_processor = self.info.get_hf_processor()
data: dict[str, object] = {}
@@ -3473,19 +3473,25 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Process each modality independently because the MiniCPM processor
asserts that image tags and image sizes have matching lengths.
"""
valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
mm_data: dict[str, object] = {}
passthrough_data: dict[str, object] = {}
for items in valid_mm_items.values():
if not items:
continue
mm_data.update(items.get_processor_data())
passthrough_data.update(items.get_passthrough_data())
tokenizer = self.info.get_tokenizer()
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs)
processed_data = BatchFeature(
{
"input_ids": input_ids,
@@ -94,9 +94,9 @@ class MiniMaxH3MultiModalProcessor(Qwen3VLMultiModalProcessor):
def _apply_hf_processor_main(
self,
mm_items: Any,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
):
base_kwargs = self._base_processor_kwargs(hf_processor_mm_kwargs)
base_kwargs = self._base_processor_kwargs(hf_kwargs)
return super()._apply_hf_processor_main(
mm_items,
base_kwargs,
@@ -14,7 +14,10 @@ from vllm.forward_context import set_forward_context
from vllm.logger import init_logger
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.worker.gpu.input_batch import get_num_sampled_and_rejected
from vllm.v1.worker.gpu.sample.bad_words import BadWordsState
from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm_omni.model_executor.output_snapshot import PackedOutputSnapshot
@@ -118,15 +121,24 @@ class MossLocalModelState(OmniModelState):
or sampler.return_sampling_mask
or sampler.trace_replay_state is not None
or sampler.get_logprobs_dims(rows) is not None
or np.any(sampler.logit_bias_state.use_logit_bias[rows])
or np.any(sampler.penalties_state.use_penalty[rows])
or np.any(sampler.bad_words_state.num_bad_words.np[rows])
or (
sampler.thinking_budget_state.enabled
and np.any(sampler.thinking_budget_state.use_thinking_budget[rows])
)
):
return None
for processor in sampler.logits_processors:
if type(processor) is LogitBiasState:
active = np.any(processor.use_logit_bias[rows])
elif type(processor) is PenaltiesState:
active = np.any(processor.use_penalty[rows])
elif type(processor) is BadWordsState:
active = np.any(processor.num_bad_words.np[rows])
else:
# Custom processors can alter even a single finite logit.
return None
if active:
return None
keep = self.model._batch_should_continue
if keep is None or keep.numel() != batch.num_reqs:
return None
@@ -28,7 +28,6 @@ from vllm.multimodal.inputs import (
)
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseProcessingInfo,
ProcessorInputs,
PromptIndexTargets,
@@ -37,7 +36,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.platforms import current_omni_platform
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
@@ -117,7 +116,7 @@ class OmniVoiceMultiModalProcessor(OmniMultiModalProcessor[OmniVoiceMultiModalPr
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Encode the reference audio, if any; the prompt was tokenized in apply().
@@ -126,7 +125,13 @@ class OmniVoiceMultiModalProcessor(OmniMultiModalProcessor[OmniVoiceMultiModalPr
touch the text tokenizer.
"""
valid_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
mm_data, passthrough = self._get_hf_mm_data(valid_items)
mm_data: dict[str, object] = {}
passthrough: dict[str, object] = {}
for items in valid_items.values():
if not items:
continue
mm_data.update(items.get_processor_data())
passthrough.update(items.get_passthrough_data())
audio = self._get_reference_audio(mm_data)
if audio is None:
return BatchFeature(dict(passthrough))
@@ -235,7 +240,7 @@ class OmniVoiceMultiModalProcessor(OmniMultiModalProcessor[OmniVoiceMultiModalPr
]
class OmniVoiceDummyInputsBuilder(BaseDummyInputsBuilder[OmniVoiceMultiModalProcessingInfo]):
class OmniVoiceDummyInputsBuilder(OmniDummyInputsBuilder[OmniVoiceMultiModalProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return "Hello, this is a test of the OmniVoice system."
@@ -78,7 +78,7 @@ from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems, Vide
from vllm.multimodal.processing.context import TimingContext
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import (
MultiModalProcessingInfo,
MultiModalProcessingResult,
MultiModalPromptUpdates,
PlaceholderFeaturesInfo,
PromptReplacement,
@@ -367,7 +367,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
"""Apply omni-specific HF kwargs before the upstream Qwen2.5-Omni
processor runs.
@@ -378,23 +378,30 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
per-video ``use_audio_in_video`` mask handling have to run here.
"""
valid_mm_items = mm_items.select({k for k, c in mm_items.get_all_counts().items() if c > 0})
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
# Upstream removed `_get_hf_mm_data`; extract the raw processor data
# (still keyed as `videos`/`video_metadata`/`audios`) for the omni
# kwargs helpers below, instead of `_get_hf_mm_inputs` which renames
# and injects a dummy text key.
processor_data: dict[str, object] = {}
for items in valid_mm_items.values():
if items:
processor_data.update(items.get_processor_data())
hf_processor_mm_kwargs = _presampled_videos_hf_kwargs(
hf_kwargs = _presampled_videos_hf_kwargs(
processor_data,
hf_processor_mm_kwargs,
hf_kwargs,
)
hf_processor_mm_kwargs = _coerce_use_audio_in_video_for_hf_processor(
hf_kwargs = _coerce_use_audio_in_video_for_hf_processor(
processor_data,
hf_processor_mm_kwargs,
hf_kwargs,
)
processed_data = super()._apply_hf_processor_main(
mm_items,
hf_processor_mm_kwargs,
hf_kwargs,
)
per_video_mask = hf_processor_mm_kwargs.get(_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY)
per_video_mask = hf_kwargs.get(_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY)
if per_video_mask is not None:
processed_data["use_audio_in_video"] = torch.tensor(per_video_mask)
return processed_data
@@ -564,32 +571,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
),
]
def _apply_hf_processor_mm_only(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
):
mm_counts = mm_items.get_all_counts()
if "video" in mm_counts:
video_use_audio_in_video = _get_request_video_use_audio_in_video(
hf_processor_mm_kwargs,
mm_counts["video"],
)
if any(video_use_audio_in_video):
assert "audio" in mm_counts
mm_counts["audio"] -= sum(video_use_audio_in_video)
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return mm_processed_data
def _get_audio_in_video_pairs(
self,
mm_data_items: MultiModalDataItems,
@@ -639,14 +620,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalProcessingInfo:
) -> MultiModalProcessingResult:
"""Cache-aware: processes video/audio pairs as units,
using pair-specific cache keys. Falls back to standard cache if no pairs."""
cache = self.cache
cache = inputs.cache
_, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
if cache is None or passthrough_data:
has_passthrough_data = any(len(items.get_passthrough_data()) > 0 for items in inputs.mm_data_items.values())
if cache is None or has_passthrough_data:
return self._apply_hf_processor(inputs, timing_ctx)
aiv_pairs = self._get_audio_in_video_pairs(
@@ -694,7 +675,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
with timing_ctx.record("apply_hf_processor"):
mm_missing_processed_data = self._apply_hf_processor_main(
mm_items=mm_missing_data_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
hf_kwargs=hf_processor_mm_kwargs,
)
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
@@ -720,7 +701,8 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
mm_missing_prompt_updates=mm_missing_prompt_updates,
)
mm_info = MultiModalProcessingInfo(
mm_info = MultiModalProcessingResult(
prompt_ids=self._postprocess_prompt(inputs.prompt),
kwargs=mm_kwargs,
hashes=mm_hashes,
prompt_updates=mm_prompt_updates,
@@ -819,11 +801,12 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates: MultiModalPromptUpdates,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_item_counts = mm_items.get_all_counts()
prompt_ids = mm_res.prompt_ids
mm_kwargs = mm_res.kwargs
mm_prompt_updates = mm_res.prompt_updates
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
@@ -561,35 +561,15 @@ class StepAudio2MultiModalProcessor(BaseMultiModalProcessor[StepAudio2Processing
)
]
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data, hf_processor_mm_kwargs = super()._preprocess_hf_mm_data(
mm_data,
hf_processor_mm_kwargs,
)
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
if isinstance(audios, str):
mm_data["audio"] = [audios]
elif isinstance(audios, (list, tuple)):
mm_data["audio"] = audios
else:
mm_data["audio"] = [audios]
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
processed_data = super()._postprocess_hf_mm_data(
mm_data,
hf_processor_mm_kwargs,
hf_data,
hf_kwargs,
processed_data,
)
if "audio_mels" not in processed_data:
@@ -30,7 +30,7 @@ class VoxCPM2OmniARAsyncScheduler(OmniARAsyncScheduler):
def _should_defer_waiting_for_unified_decode_graph(self) -> bool:
if not self._unified_decode_graph_enabled():
return False
if not self.waiting or not self.running:
if not (self.waiting or self.kv_holding_waiting) or not self.running:
return False
for request in self.running:
@@ -48,6 +48,8 @@ from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems, Mult
from vllm.multimodal.processing import BaseDummyInputsBuilder, BaseMultiModalProcessor
from vllm.multimodal.processing.processor import (
BaseProcessingInfo,
MultiModalProcessingResult,
PlaceholderFeaturesInfo,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
@@ -836,6 +838,14 @@ class VoxtralTTSDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralTTSProcessingIn
class VoxtralTTSMultiModalProcessor(BaseMultiModalProcessor[VoxtralTTSProcessingInfo]):
def get_dummy_inputs(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, Any],
) -> ProcessorInputs:
return self.dummy_inputs.get_dummy_processor_inputs(seq_len, mm_counts, mm_options)
def _get_mm_fields_config(
self,
hf_inputs: Mapping[str, NestedTensors],
@@ -874,39 +884,17 @@ class VoxtralTTSMultiModalProcessor(BaseMultiModalProcessor[VoxtralTTSProcessing
),
]
def _apply_hf_processor_mm_only(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> BatchFeature:
"""
Apply the HF processor on the multi-modal data only.
Issue: Voxtral TTS use Mistral Tokenizer with custom audio encoder. It doesn't
inherit Transformers ProcessorMixin and can't use call_hf_processor_mm_only.
Solution: Override this method to call _apply_hf_processor_text_mm directly.
"""
mm_counts = mm_items.get_all_counts()
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
)
return mm_processed_data
def _maybe_apply_prompt_updates(
self,
mm_items: MultiModalDataItems,
prompt_ids: list[int],
mm_kwargs: MultiModalKwargsItems,
mm_prompt_updates,
):
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
# Voxtral's Mistral chat template has already inserted the complete
# audio-token run, so locate it without applying the replacement again.
mm_item_counts = mm_items.get_all_counts()
prompt_ids = mm_res.prompt_ids
mm_kwargs = mm_res.kwargs
mm_prompt_updates = mm_res.prompt_updates
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
mm_placeholders = self._find_mm_placeholders(