From 8e950daf6a09ecfac1c612b89b4aa42a8f82f89b Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Thu, 26 Feb 2026 09:26:25 +0800 Subject: [PATCH] [Debug] Multi-Request for Qwen 3 Omni use_audio_in_video (#1433) Signed-off-by: tzhouam --- .../models/qwen2_5_omni/qwen2_5_omni.py | 6 +- .../qwen2_5_omni/qwen2_5_omni_talker.py | 8 +- .../qwen2_5_omni/qwen2_5_omni_thinker.py | 75 ++++++++++++++++++- .../models/qwen3_omni/qwen3_omni.py | 8 +- .../qwen3_omni/qwen3_omni_moe_thinker.py | 16 ++-- 5 files changed, 98 insertions(+), 15 deletions(-) diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py index 936a18427..39baa5573 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py @@ -17,8 +17,6 @@ from vllm.logger import init_logger from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP from vllm.model_executor.models.qwen2_5_omni_thinker import ( Qwen2_5OmniConditionalGenerationMixin, - Qwen2_5OmniThinkerDummyInputsBuilder, - Qwen2_5OmniThinkerMultiModalProcessor, Qwen2_5OmniThinkerProcessingInfo, ) from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix @@ -37,6 +35,10 @@ from vllm.v1.sample.sampler import Sampler from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific from vllm_omni.model_executor.models.output_templates import OmniOutput +from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( + Qwen2_5OmniThinkerDummyInputsBuilder, + Qwen2_5OmniThinkerMultiModalProcessor, +) from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges from vllm_omni.platforms import current_omni_platform diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py index 927bc5525..988d8c236 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_talker.py @@ -11,8 +11,6 @@ from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP from vllm.model_executor.models.qwen2_5_omni_thinker import ( - Qwen2_5OmniThinkerDummyInputsBuilder, - Qwen2_5OmniThinkerMultiModalProcessor, Qwen2_5OmniThinkerProcessingInfo, ) from vllm.model_executor.models.qwen2_5_vl import Qwen2_5_VisionTransformer @@ -28,7 +26,11 @@ from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.sample.sampler import Sampler -from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import Qwen2_5OmniConditionalGenerationMixin +from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( + Qwen2_5OmniConditionalGenerationMixin, + Qwen2_5OmniThinkerDummyInputsBuilder, + Qwen2_5OmniThinkerMultiModalProcessor, +) @MULTIMODAL_REGISTRY.register_processor( diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index 8ee516376..a4c8dcab5 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -1,6 +1,6 @@ """Thin Omni wrapper: reuse upstream Qwen2.5-Omni thinker (v0.14) with minimal overrides.""" -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any import torch @@ -25,7 +25,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.qwen2_5_omni_thinker import ( Qwen2_5OmniAudioFeatureInputs, Qwen2_5OmniThinkerDummyInputsBuilder, - Qwen2_5OmniThinkerMultiModalProcessor, Qwen2_5OmniThinkerProcessingInfo, check_interleaved_audio_video, merge_interleaved_embeddings, @@ -33,6 +32,9 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import ( from vllm.model_executor.models.qwen2_5_omni_thinker import ( Qwen2_5OmniConditionalGenerationMixin as Qwen2_5OmniConditionalGenerationMixinBase, ) +from vllm.model_executor.models.qwen2_5_omni_thinker import ( + Qwen2_5OmniThinkerMultiModalProcessor as _Qwen2_5OmniThinkerMultiModalProcessorBase, +) from vllm.model_executor.models.qwen2_5_vl import ( Qwen2_5_VisionTransformer, Qwen2_5_VLImageEmbeddingInputs, @@ -54,6 +56,12 @@ from vllm.model_executor.models.vision import get_llm_pos_ids_for_vision from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFeatureSpec, + MultiModalKwargsItems, +) +from vllm.multimodal.parse import MultiModalDataItems +from vllm.multimodal.processing.processor import ( + MultiModalPromptUpdates, + PlaceholderFeaturesInfo, ) from vllm.sequence import IntermediateTensors @@ -64,6 +72,69 @@ except (ImportError, ModuleNotFoundError): logger = init_logger(__name__) +class Qwen2_5OmniThinkerMultiModalProcessor( + _Qwen2_5OmniThinkerMultiModalProcessorBase, +): + """Override to fix use_audio_in_video detection when mm cache returns None.""" + + def _maybe_apply_prompt_updates( + self, + mm_items: MultiModalDataItems, + prompt_ids: list[int], + mm_kwargs: MultiModalKwargsItems, + mm_prompt_updates: MultiModalPromptUpdates, + is_update_applied: bool, + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + mm_item_counts = mm_items.get_all_counts() + self._validate_mm_kwargs(mm_kwargs, mm_item_counts) + self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + + use_audio_in_video = False + if "video" in mm_kwargs: + non_none_items = [item for item in mm_kwargs["video"] if item is not None] + if non_none_items: + for item in non_none_items: + if item.get("use_audio_in_video"): + uaiv_tensor = item["use_audio_in_video"].data + if uaiv_tensor.numel() > 0: + use_audio_in_video = bool(uaiv_tensor.item()) + break + elif "audio" in mm_prompt_updates: + tokenizer = self.info.get_tokenizer() + audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>") + use_audio_in_video = audio_pad_id not in prompt_ids + + if is_update_applied: + mm_placeholders = self._find_mm_placeholders( + prompt_ids, + mm_prompt_updates, + ) + self._validate_mm_placeholders( + mm_placeholders, + mm_item_counts, + ) + else: + if use_audio_in_video and "audio" in mm_prompt_updates: + filtered_updates = {k: v for k, v in mm_prompt_updates.items() if k != "audio"} + prompt_ids, mm_placeholders = self._apply_prompt_updates( + prompt_ids, + filtered_updates, + ) + mm_placeholders = self._derive_audio_from_video_placeholders(mm_placeholders, mm_prompt_updates) + else: + prompt_ids, mm_placeholders = self._apply_prompt_updates( + prompt_ids, + mm_prompt_updates, + ) + + self._validate_mm_placeholders( + mm_placeholders, + mm_item_counts, + ) + + return prompt_ids, mm_placeholders + + class Qwen2_5OmniConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMixinBase): def _parse_and_validate_audio_input(self, **kwargs: object) -> Qwen2_5OmniAudioFeatureInputs | None: input_audio_features = kwargs.pop("input_audio_features", None) diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py index 58eb1ff22..32137e9ae 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py @@ -20,9 +20,6 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP from vllm.model_executor.models.qwen3_omni_moe_thinker import ( Qwen3OmniMoeConditionalGenerationMixin, - Qwen3OmniMoeThinkerDummyInputsBuilder, - Qwen3OmniMoeThinkerMultiModalProcessor, - Qwen3OmniMoeThinkerProcessingInfo, ) from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix from vllm.multimodal import MULTIMODAL_REGISTRY @@ -34,6 +31,11 @@ from vllm.v1.sample.sampler import Sampler from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin from vllm_omni.model_executor.models.output_templates import OmniOutput +from vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker import ( + Qwen3OmniMoeThinkerDummyInputsBuilder, + Qwen3OmniMoeThinkerMultiModalProcessor, + Qwen3OmniMoeThinkerProcessingInfo, +) from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, safe_tensor_reshape from vllm_omni.platforms import current_omni_platform diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py index a0920c069..f509b4aa5 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py @@ -343,11 +343,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( use_audio_in_video = False if "video" in mm_kwargs: - for item in mm_kwargs["video"]: - if item and item["use_audio_in_video"].data: - use_audio_in_video = True - else: - use_audio_in_video = False + non_none_items = [item for item in mm_kwargs["video"] if item is not None] + if non_none_items: + # Normal case: at least one non-cached item, read flag directly + use_audio_in_video = any(item["use_audio_in_video"].data for item in non_none_items) + elif "audio" in mm_prompt_updates: + # All video items are from cache (None); infer from prompt: + # use_audio_in_video=True means the prompt has no <|audio_pad|> + # placeholder (audio is embedded in video tokens instead) + tokenizer = self.info.get_tokenizer() + audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>") + use_audio_in_video = audio_pad_id not in prompt_ids # normal case with `use_audio_in_video=False` if is_update_applied: