[Rebase] Align vLLM-Omni with vLLM v0.31.0 (#8459)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
1f47fa2f29
commit
0b0d2d7009
141 files changed
+2885
-721
No files matched your search
@@ -259,19 +259,25 @@ class OmniBagelMultiModalProcessor(BaseMultiModalProcessor[OmniBagelProcessingIn
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
mm_data: dict[str, object] = {}
|
||||
passthrough_data: dict[str, object] = {}
|
||||
for items in valid_mm_items.values():
|
||||
if not items:
|
||||
continue
|
||||
mm_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
processor = self.info.get_hf_processor(**hf_kwargs)
|
||||
|
||||
has_image = "images" in mm_data
|
||||
has_img2img = "pixel_values_img2img" in mm_data
|
||||
processed_data = BatchFeature()
|
||||
|
||||
if has_image:
|
||||
image_kwargs = {**hf_processor_mm_kwargs, "is_img2img": False}
|
||||
image_kwargs = {**hf_kwargs, "is_img2img": False}
|
||||
image_outputs = self.info.ctx.call_hf_processor(
|
||||
processor,
|
||||
{"text": prompt_text, "images": mm_data["images"]},
|
||||
@@ -280,7 +286,7 @@ class OmniBagelMultiModalProcessor(BaseMultiModalProcessor[OmniBagelProcessingIn
|
||||
processed_data.update(image_outputs)
|
||||
|
||||
if has_img2img:
|
||||
img2img_kwargs = self._mm_kwargs_for_bagel_img2img_hf(hf_processor_mm_kwargs)
|
||||
img2img_kwargs = self._mm_kwargs_for_bagel_img2img_hf(hf_kwargs)
|
||||
img2img_kwargs["is_img2img"] = True
|
||||
img2img_outputs = self.info.ctx.call_hf_processor(
|
||||
processor,
|
||||
|
||||
@@ -89,14 +89,12 @@ def _cfg_extra(request: Any, key: str) -> Any:
|
||||
|
||||
|
||||
def _wait_queues(scheduler: Any) -> list[Any]:
|
||||
# Divergence from the official (vLLM 0.20-era) file: vLLM 0.24 parks
|
||||
# structurally not-ready requests in a second ``skipped_waiting`` queue;
|
||||
# pair-hold must cover both queues or a lone member parked there would
|
||||
# be admitted without its partner.
|
||||
# Pair holding must cover requests that retain KV blocks as well as
|
||||
# fresh requests, or a lone member can be admitted without its partner.
|
||||
queues = [scheduler.waiting]
|
||||
skipped = getattr(scheduler, "skipped_waiting", None)
|
||||
if skipped is not None:
|
||||
queues.append(skipped)
|
||||
kv_holding = getattr(scheduler, "kv_holding_waiting", None)
|
||||
if kv_holding is not None:
|
||||
queues.append(kv_holding)
|
||||
return queues
|
||||
|
||||
|
||||
@@ -108,7 +106,7 @@ def _pair_complete(scheduler: Any, request_id: str, blocked_ids: set[str] | None
|
||||
if len(roles) != 2 or not all(rid in scheduler.requests for rid in roles.values()):
|
||||
return False
|
||||
if blocked_ids:
|
||||
# A partner parked in skipped_waiting (e.g. waiting for remote KVs)
|
||||
# A deferred partner (e.g. waiting for remote KVs)
|
||||
# may fail promotion this step while this member gets scheduled from
|
||||
# waiting; hold this member until the partner is schedulable.
|
||||
for rid in roles.values():
|
||||
@@ -149,14 +147,14 @@ def _hold_incomplete_pairs(scheduler: Any) -> list[tuple[Any, Any]]:
|
||||
hold_counts: dict[str, int] = getattr(scheduler, "_cfg_hold_counts", None) or {}
|
||||
scheduler._cfg_hold_counts = hold_counts
|
||||
|
||||
skipped = getattr(scheduler, "skipped_waiting", None)
|
||||
blocked_ids = {req.request_id for req in list(skipped)} if skipped is not None else set()
|
||||
kv_holding = getattr(scheduler, "kv_holding_waiting", None)
|
||||
blocked_ids = {req.request_id for req in getattr(scheduler, "deferred_waiting", ())}
|
||||
|
||||
held: list[tuple[Any, Any]] = []
|
||||
for queue in _wait_queues(scheduler):
|
||||
# A member is only "blocked" for its partner's sake when it sits in
|
||||
# the OTHER queue; members of the queue being scanned move together.
|
||||
partner_blockers = blocked_ids if queue is not skipped else set()
|
||||
partner_blockers = blocked_ids if queue is not kv_holding else set()
|
||||
held_here = []
|
||||
for request in list(queue):
|
||||
if _pair_complete(scheduler, request.request_id, partner_blockers):
|
||||
@@ -181,8 +179,12 @@ def _release_held(held: list[tuple[Any, Any]]) -> None:
|
||||
|
||||
|
||||
def _reorder_waiting_for_cfg(scheduler: Any) -> None:
|
||||
"""Move CFG pair partners adjacent in the FCFS waiting queue."""
|
||||
waiting = scheduler.waiting
|
||||
"""Move CFG pair partners adjacent in each FCFS waiting queue."""
|
||||
for waiting in _wait_queues(scheduler):
|
||||
_reorder_cfg_queue(scheduler, waiting)
|
||||
|
||||
|
||||
def _reorder_cfg_queue(scheduler: Any, waiting: Any) -> None:
|
||||
# Only the FCFS queue (a deque) has caller-controlled ordering; priority
|
||||
# queues order themselves and pair sync then relies on hold + equalize.
|
||||
if not isinstance(waiting, deque) or len(waiting) < 2:
|
||||
|
||||
@@ -28,7 +28,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptIndexTargets,
|
||||
@@ -42,7 +41,7 @@ from vllm.v1.sample.ops.topk_topp_sampler import random_sample
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.data_entry_keys import EmbeddingsStruct, OmniPayloadStruct, to_dict, to_struct
|
||||
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
|
||||
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
|
||||
from vllm_omni.model_executor.models.cosyvoice3.ras_sampler import MAX_FUSED_TOP_K, fused_ras_sample
|
||||
from vllm_omni.model_executor.models.cosyvoice3.runtime import (
|
||||
cosyvoice3_batch_flow_debug,
|
||||
@@ -696,7 +695,7 @@ class CosyVoice3MultiModalProcessor(OmniMultiModalProcessor[CosyVoice3MultiModal
|
||||
]
|
||||
|
||||
|
||||
class CosyVoice3DummyInputsBuilder(BaseDummyInputsBuilder[CosyVoice3MultiModalProcessingInfo]):
|
||||
class CosyVoice3DummyInputsBuilder(OmniDummyInputsBuilder[CosyVoice3MultiModalProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return "Hello, this is a test of the CosyVoice3 system capability."
|
||||
|
||||
|
||||
@@ -459,16 +459,22 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Process source images under vLLM's token-only prompt contract."""
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
num_images = mm_counts.get("image", 0)
|
||||
if num_images == 0:
|
||||
return BatchFeature({"mrope_image_grid_thw": self._build_generation_grids(hf_processor_mm_kwargs)})
|
||||
return BatchFeature({"mrope_image_grid_thw": self._build_generation_grids(hf_kwargs)})
|
||||
|
||||
valid_mm_items = mm_items.select({key for key, count in mm_counts.items() if count > 0})
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
processor_data: dict[str, object] = {}
|
||||
passthrough_data: dict[str, object] = {}
|
||||
for items in valid_mm_items.values():
|
||||
if not items:
|
||||
continue
|
||||
processor_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
images = processor_data.get("images")
|
||||
if not images:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
@@ -483,7 +489,7 @@ class GlmImageMultiModalProcessor(BaseMultiModalProcessor[GlmImageProcessingInfo
|
||||
source_grid_thw = image_grid_thw[:num_images]
|
||||
image_inputs["image_grid_thw"] = source_grid_thw
|
||||
|
||||
target_grid = self._build_generation_grids(hf_processor_mm_kwargs)[:1].to(dtype=source_grid_thw.dtype)
|
||||
target_grid = self._build_generation_grids(hf_kwargs)[:1].to(dtype=source_grid_thw.dtype)
|
||||
image_inputs["mrope_image_grid_thw"] = torch.cat(
|
||||
[source_grid_thw, target_grid],
|
||||
dim=0,
|
||||
|
||||
@@ -38,7 +38,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptIndexTargets,
|
||||
@@ -50,7 +49,7 @@ from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
|
||||
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
|
||||
from vllm_omni.model_executor.models.common.nucleus_ras_sampling import ras_sample_one as _ras_sample_one
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -674,7 +673,7 @@ class GLMTTSMultiModalProcessor(OmniMultiModalProcessor[GLMTTSMultiModalProcessi
|
||||
]
|
||||
|
||||
|
||||
class GLMTTSDummyInputsBuilder(BaseDummyInputsBuilder[GLMTTSMultiModalProcessingInfo]):
|
||||
class GLMTTSDummyInputsBuilder(OmniDummyInputsBuilder[GLMTTSMultiModalProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return "This is a test of the GLM-TTS voice cloning system."
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ from vllm.multimodal.inputs import (
|
||||
)
|
||||
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
@@ -44,7 +43,7 @@ from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
|
||||
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.models.mimo_audio.config_mimo_audio import (
|
||||
NO_INTERLEAVE_NEXT_TOKEN_ID,
|
||||
@@ -224,7 +223,7 @@ class MiMoAudioLLMProcessingInfo(
|
||||
return {"audio": 1}
|
||||
|
||||
|
||||
class MiMoAudioLLMDummyInputsBuilder(BaseDummyInputsBuilder[MiMoAudioLLMProcessingInfo]):
|
||||
class MiMoAudioLLMDummyInputsBuilder(OmniDummyInputsBuilder[MiMoAudioLLMProcessingInfo]):
|
||||
_processor_inputs_cache: LRUCache = LRUCache(capacity=1024)
|
||||
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
@@ -372,7 +371,7 @@ class MiMoAudioLLMMultiModalProcessor(OmniMultiModalProcessor[MiMoAudioLLMProces
|
||||
|
||||
# Text-only input not supported in composite processor
|
||||
if not mm_data.get("audio", []):
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
prompt_ids = self._postprocess_prompt(prompt_ids)
|
||||
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
|
||||
@@ -396,25 +396,21 @@ class MimoAudioDummyInputsBuilder(BaseDummyInputsBuilder[MimoAudioProcessingInfo
|
||||
|
||||
|
||||
class MimoAudioMultiModalProcessor(BaseMultiModalProcessor[MimoAudioProcessingInfo]):
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data, hf_processor_mm_kwargs = super()._preprocess_hf_mm_data(
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
mm_data["audio"] = audios
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
|
||||
hf_processor_mm_kwargs = {
|
||||
**hf_processor_mm_kwargs,
|
||||
"sampling_rate": feature_extractor.sampling_rate,
|
||||
}
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
):
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
if hf_inputs.hf_data.get("audio"):
|
||||
feature_extractor = self.info.get_feature_extractor(**hf_kwargs)
|
||||
hf_inputs = hf_inputs._replace(
|
||||
hf_kwargs={
|
||||
**hf_kwargs,
|
||||
"sampling_rate": feature_extractor.sampling_rate,
|
||||
},
|
||||
)
|
||||
return hf_inputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
@@ -516,7 +516,7 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Call Ming's modality processors without expanding prompt text.
|
||||
|
||||
@@ -525,7 +525,13 @@ class MingFlashOmniThinkerMultiModalProcessor(BaseMultiModalProcessor[MingFlashO
|
||||
placeholder tokens remain under vLLM's prompt-update control.
|
||||
"""
|
||||
valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
mm_data: dict[str, object] = {}
|
||||
passthrough_data: dict[str, object] = {}
|
||||
for items in valid_mm_items.values():
|
||||
if not items:
|
||||
continue
|
||||
mm_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
|
||||
data: dict[str, object] = {}
|
||||
|
||||
@@ -3473,19 +3473,25 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Process each modality independently because the MiniCPM processor
|
||||
asserts that image tags and image sizes have matching lengths.
|
||||
"""
|
||||
valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
mm_data: dict[str, object] = {}
|
||||
passthrough_data: dict[str, object] = {}
|
||||
for items in valid_mm_items.values():
|
||||
if not items:
|
||||
continue
|
||||
mm_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||
mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs)
|
||||
processed_data = BatchFeature(
|
||||
{
|
||||
"input_ids": input_ids,
|
||||
|
||||
@@ -94,9 +94,9 @@ class MiniMaxH3MultiModalProcessor(Qwen3VLMultiModalProcessor):
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: Any,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
):
|
||||
base_kwargs = self._base_processor_kwargs(hf_processor_mm_kwargs)
|
||||
base_kwargs = self._base_processor_kwargs(hf_kwargs)
|
||||
return super()._apply_hf_processor_main(
|
||||
mm_items,
|
||||
base_kwargs,
|
||||
|
||||
@@ -14,7 +14,10 @@ from vllm.forward_context import set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.worker.gpu.input_batch import get_num_sampled_and_rejected
|
||||
from vllm.v1.worker.gpu.sample.bad_words import BadWordsState
|
||||
from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.output_snapshot import PackedOutputSnapshot
|
||||
@@ -118,15 +121,24 @@ class MossLocalModelState(OmniModelState):
|
||||
or sampler.return_sampling_mask
|
||||
or sampler.trace_replay_state is not None
|
||||
or sampler.get_logprobs_dims(rows) is not None
|
||||
or np.any(sampler.logit_bias_state.use_logit_bias[rows])
|
||||
or np.any(sampler.penalties_state.use_penalty[rows])
|
||||
or np.any(sampler.bad_words_state.num_bad_words.np[rows])
|
||||
or (
|
||||
sampler.thinking_budget_state.enabled
|
||||
and np.any(sampler.thinking_budget_state.use_thinking_budget[rows])
|
||||
)
|
||||
):
|
||||
return None
|
||||
for processor in sampler.logits_processors:
|
||||
if type(processor) is LogitBiasState:
|
||||
active = np.any(processor.use_logit_bias[rows])
|
||||
elif type(processor) is PenaltiesState:
|
||||
active = np.any(processor.use_penalty[rows])
|
||||
elif type(processor) is BadWordsState:
|
||||
active = np.any(processor.num_bad_words.np[rows])
|
||||
else:
|
||||
# Custom processors can alter even a single finite logit.
|
||||
return None
|
||||
if active:
|
||||
return None
|
||||
keep = self.model._batch_should_continue
|
||||
if keep is None or keep.numel() != batch.num_reqs:
|
||||
return None
|
||||
|
||||
@@ -28,7 +28,6 @@ from vllm.multimodal.inputs import (
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptIndexTargets,
|
||||
@@ -37,7 +36,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.inputs.mm_processor import OmniMultiModalProcessor
|
||||
from vllm_omni.inputs.mm_processor import OmniDummyInputsBuilder, OmniMultiModalProcessor
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
from vllm_omni.transformers_utils.configs.omnivoice import OmniVoiceConfig
|
||||
@@ -117,7 +116,7 @@ class OmniVoiceMultiModalProcessor(OmniMultiModalProcessor[OmniVoiceMultiModalPr
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Encode the reference audio, if any; the prompt was tokenized in apply().
|
||||
|
||||
@@ -126,7 +125,13 @@ class OmniVoiceMultiModalProcessor(OmniMultiModalProcessor[OmniVoiceMultiModalPr
|
||||
touch the text tokenizer.
|
||||
"""
|
||||
valid_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
|
||||
mm_data, passthrough = self._get_hf_mm_data(valid_items)
|
||||
mm_data: dict[str, object] = {}
|
||||
passthrough: dict[str, object] = {}
|
||||
for items in valid_items.values():
|
||||
if not items:
|
||||
continue
|
||||
mm_data.update(items.get_processor_data())
|
||||
passthrough.update(items.get_passthrough_data())
|
||||
audio = self._get_reference_audio(mm_data)
|
||||
if audio is None:
|
||||
return BatchFeature(dict(passthrough))
|
||||
@@ -235,7 +240,7 @@ class OmniVoiceMultiModalProcessor(OmniMultiModalProcessor[OmniVoiceMultiModalPr
|
||||
]
|
||||
|
||||
|
||||
class OmniVoiceDummyInputsBuilder(BaseDummyInputsBuilder[OmniVoiceMultiModalProcessingInfo]):
|
||||
class OmniVoiceDummyInputsBuilder(OmniDummyInputsBuilder[OmniVoiceMultiModalProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return "Hello, this is a test of the OmniVoice system."
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems, Vide
|
||||
from vllm.multimodal.processing.context import TimingContext
|
||||
from vllm.multimodal.processing.inputs import ProcessorInputs
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessingResult,
|
||||
MultiModalPromptUpdates,
|
||||
PlaceholderFeaturesInfo,
|
||||
PromptReplacement,
|
||||
@@ -367,7 +367,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Apply omni-specific HF kwargs before the upstream Qwen2.5-Omni
|
||||
processor runs.
|
||||
@@ -378,23 +378,30 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
per-video ``use_audio_in_video`` mask handling have to run here.
|
||||
"""
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_items.get_all_counts().items() if c > 0})
|
||||
processor_data, _ = self._get_hf_mm_data(valid_mm_items)
|
||||
# Upstream removed `_get_hf_mm_data`; extract the raw processor data
|
||||
# (still keyed as `videos`/`video_metadata`/`audios`) for the omni
|
||||
# kwargs helpers below, instead of `_get_hf_mm_inputs` which renames
|
||||
# and injects a dummy text key.
|
||||
processor_data: dict[str, object] = {}
|
||||
for items in valid_mm_items.values():
|
||||
if items:
|
||||
processor_data.update(items.get_processor_data())
|
||||
|
||||
hf_processor_mm_kwargs = _presampled_videos_hf_kwargs(
|
||||
hf_kwargs = _presampled_videos_hf_kwargs(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
hf_processor_mm_kwargs = _coerce_use_audio_in_video_for_hf_processor(
|
||||
hf_kwargs = _coerce_use_audio_in_video_for_hf_processor(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
processed_data = super()._apply_hf_processor_main(
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
per_video_mask = hf_processor_mm_kwargs.get(_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY)
|
||||
per_video_mask = hf_kwargs.get(_PER_VIDEO_USE_AUDIO_IN_VIDEO_KEY)
|
||||
if per_video_mask is not None:
|
||||
processed_data["use_audio_in_video"] = torch.tensor(per_video_mask)
|
||||
return processed_data
|
||||
@@ -564,32 +571,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
),
|
||||
]
|
||||
|
||||
def _apply_hf_processor_mm_only(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
):
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
if "video" in mm_counts:
|
||||
video_use_audio_in_video = _get_request_video_use_audio_in_video(
|
||||
hf_processor_mm_kwargs,
|
||||
mm_counts["video"],
|
||||
)
|
||||
if any(video_use_audio_in_video):
|
||||
assert "audio" in mm_counts
|
||||
mm_counts["audio"] -= sum(video_use_audio_in_video)
|
||||
|
||||
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return mm_processed_data
|
||||
|
||||
def _get_audio_in_video_pairs(
|
||||
self,
|
||||
mm_data_items: MultiModalDataItems,
|
||||
@@ -639,14 +620,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
"""Cache-aware: processes video/audio pairs as units,
|
||||
using pair-specific cache keys. Falls back to standard cache if no pairs."""
|
||||
|
||||
cache = self.cache
|
||||
cache = inputs.cache
|
||||
|
||||
_, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
|
||||
if cache is None or passthrough_data:
|
||||
has_passthrough_data = any(len(items.get_passthrough_data()) > 0 for items in inputs.mm_data_items.values())
|
||||
if cache is None or has_passthrough_data:
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
aiv_pairs = self._get_audio_in_video_pairs(
|
||||
@@ -694,7 +675,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
mm_missing_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
hf_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
@@ -720,7 +701,8 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
mm_missing_prompt_updates=mm_missing_prompt_updates,
|
||||
)
|
||||
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
mm_info = MultiModalProcessingResult(
|
||||
prompt_ids=self._postprocess_prompt(inputs.prompt),
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
@@ -819,11 +801,12 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
prompt_ids = mm_res.prompt_ids
|
||||
mm_kwargs = mm_res.kwargs
|
||||
mm_prompt_updates = mm_res.prompt_updates
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
|
||||
|
||||
|
||||
@@ -561,35 +561,15 @@ class StepAudio2MultiModalProcessor(BaseMultiModalProcessor[StepAudio2Processing
|
||||
)
|
||||
]
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data, hf_processor_mm_kwargs = super()._preprocess_hf_mm_data(
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
if audios:
|
||||
if isinstance(audios, str):
|
||||
mm_data["audio"] = [audios]
|
||||
elif isinstance(audios, (list, tuple)):
|
||||
mm_data["audio"] = audios
|
||||
else:
|
||||
mm_data["audio"] = [audios]
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
processed_data = super()._postprocess_hf_mm_data(
|
||||
mm_data,
|
||||
hf_processor_mm_kwargs,
|
||||
hf_data,
|
||||
hf_kwargs,
|
||||
processed_data,
|
||||
)
|
||||
if "audio_mels" not in processed_data:
|
||||
|
||||
@@ -30,7 +30,7 @@ class VoxCPM2OmniARAsyncScheduler(OmniARAsyncScheduler):
|
||||
def _should_defer_waiting_for_unified_decode_graph(self) -> bool:
|
||||
if not self._unified_decode_graph_enabled():
|
||||
return False
|
||||
if not self.waiting or not self.running:
|
||||
if not (self.waiting or self.kv_holding_waiting) or not self.running:
|
||||
return False
|
||||
|
||||
for request in self.running:
|
||||
|
||||
@@ -48,6 +48,8 @@ from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems, Mult
|
||||
from vllm.multimodal.processing import BaseDummyInputsBuilder, BaseMultiModalProcessor
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseProcessingInfo,
|
||||
MultiModalProcessingResult,
|
||||
PlaceholderFeaturesInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
@@ -836,6 +838,14 @@ class VoxtralTTSDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralTTSProcessingIn
|
||||
|
||||
|
||||
class VoxtralTTSMultiModalProcessor(BaseMultiModalProcessor[VoxtralTTSProcessingInfo]):
|
||||
def get_dummy_inputs(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, Any],
|
||||
) -> ProcessorInputs:
|
||||
return self.dummy_inputs.get_dummy_processor_inputs(seq_len, mm_counts, mm_options)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: Mapping[str, NestedTensors],
|
||||
@@ -874,39 +884,17 @@ class VoxtralTTSMultiModalProcessor(BaseMultiModalProcessor[VoxtralTTSProcessing
|
||||
),
|
||||
]
|
||||
|
||||
def _apply_hf_processor_mm_only(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Apply the HF processor on the multi-modal data only.
|
||||
|
||||
Issue: Voxtral TTS use Mistral Tokenizer with custom audio encoder. It doesn't
|
||||
inherit Transformers ProcessorMixin and can't use call_hf_processor_mm_only.
|
||||
|
||||
Solution: Override this method to call _apply_hf_processor_text_mm directly.
|
||||
"""
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
return mm_processed_data
|
||||
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsItems,
|
||||
mm_prompt_updates,
|
||||
):
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
# Voxtral's Mistral chat template has already inserted the complete
|
||||
# audio-token run, so locate it without applying the replacement again.
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
prompt_ids = mm_res.prompt_ids
|
||||
mm_kwargs = mm_res.kwargs
|
||||
mm_prompt_updates = mm_res.prompt_updates
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
|
||||
mm_placeholders = self._find_mm_placeholders(
|
||||
|
||||
Reference in new issue
Block a user