[Bugfix][Model] Fix Qwen3-Omni video metadata handling for use_audio_in_video (#4959)
Signed-off-by: ZhengWG <zwg0606@gmail.com>
This commit is contained in:
1 parent
020792e6be
commit
9fe26cfacb
4 files changed
+206
-7
No files matched your search
@@ -410,6 +410,44 @@ def test_audio_in_video_002(omni_server, openai_client) -> None:
|
|||||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||||
|
|
||||||
|
|
||||||
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
|
def test_audio_in_video_default_loader_sampling_regression(omni_server, openai_client) -> None:
|
||||||
|
"""
|
||||||
|
Regression: ``use_audio_in_video`` with default video loader sampling (no
|
||||||
|
``media_io_kwargs.video.fps``) after a prior request that pinned fps.
|
||||||
|
|
||||||
|
Before the fix the second request could partial-hit mm-cache (audio cached,
|
||||||
|
video re-processed without audio) and fail with ``StopIteration`` in
|
||||||
|
``replace_multimodal_special_tokens``.
|
||||||
|
"""
|
||||||
|
video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 128, embed_audio=True)['base64']}"
|
||||||
|
messages = dummy_messages_from_mix_data(
|
||||||
|
system_prompt=get_system_prompt(),
|
||||||
|
video_data_url=video_data_url,
|
||||||
|
content_text="Reply with one word: OK.",
|
||||||
|
)
|
||||||
|
|
||||||
|
base_config = {
|
||||||
|
"model": omni_server.model,
|
||||||
|
"messages": messages,
|
||||||
|
"stream": False,
|
||||||
|
"use_audio_in_video": True,
|
||||||
|
"modalities": ["text"],
|
||||||
|
}
|
||||||
|
|
||||||
|
# Prime mm-cache with explicit loader fps (historical failure mode).
|
||||||
|
openai_client.send_omni_request(
|
||||||
|
{
|
||||||
|
**base_config,
|
||||||
|
"extra_body": {"media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}}},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Default loader sampling: must succeed without explicit media_io fps.
|
||||||
|
openai_client.send_omni_request(dict(base_config))
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
def test_one_word_prompt_001(omni_server, openai_client) -> None:
|
def test_one_word_prompt_001(omni_server, openai_client) -> None:
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ from vllm.logger import init_logger
|
|||||||
from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP
|
from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal, SupportsPP
|
||||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||||
Qwen2_5OmniConditionalGenerationMixin,
|
Qwen2_5OmniConditionalGenerationMixin,
|
||||||
Qwen2_5OmniThinkerProcessingInfo,
|
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
|
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
|
||||||
from vllm.model_executor.models.vision import (
|
from vllm.model_executor.models.vision import (
|
||||||
@@ -39,6 +38,7 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
|
|||||||
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
|
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
|
||||||
Qwen2_5OmniThinkerDummyInputsBuilder,
|
Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||||
|
Qwen2_5OmniThinkerProcessingInfo,
|
||||||
)
|
)
|
||||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges
|
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges
|
||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from collections.abc import Iterable, Iterator, Mapping
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
import PIL.Image as PILImage
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
|
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
|
||||||
@@ -26,16 +27,21 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
|||||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||||
Qwen2_5OmniAudioFeatureInputs,
|
Qwen2_5OmniAudioFeatureInputs,
|
||||||
Qwen2_5OmniThinkerDummyInputsBuilder,
|
Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||||
Qwen2_5OmniThinkerProcessingInfo,
|
|
||||||
check_interleaved_audio_video,
|
check_interleaved_audio_video,
|
||||||
merge_interleaved_embeddings,
|
merge_interleaved_embeddings,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||||
Qwen2_5OmniConditionalGenerationMixin as Qwen2_5OmniConditionalGenerationMixinBase,
|
Qwen2_5OmniConditionalGenerationMixin as Qwen2_5OmniConditionalGenerationMixinBase,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||||
|
Qwen2_5OmniThinkerMultiModalDataParser as _Qwen2_5OmniThinkerMultiModalDataParserBase,
|
||||||
|
)
|
||||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||||
Qwen2_5OmniThinkerMultiModalProcessor as _Qwen2_5OmniThinkerMultiModalProcessorBase,
|
Qwen2_5OmniThinkerMultiModalProcessor as _Qwen2_5OmniThinkerMultiModalProcessorBase,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||||
|
Qwen2_5OmniThinkerProcessingInfo as _Qwen2_5OmniThinkerProcessingInfoBase,
|
||||||
|
)
|
||||||
from vllm.model_executor.models.qwen2_5_vl import (
|
from vllm.model_executor.models.qwen2_5_vl import (
|
||||||
Qwen2_5_VisionTransformer,
|
Qwen2_5_VisionTransformer,
|
||||||
Qwen2_5_VLImageEmbeddingInputs,
|
Qwen2_5_VLImageEmbeddingInputs,
|
||||||
@@ -56,12 +62,13 @@ from vllm.multimodal.inputs import (
|
|||||||
MultiModalFeatureSpec,
|
MultiModalFeatureSpec,
|
||||||
MultiModalKwargsItems,
|
MultiModalKwargsItems,
|
||||||
)
|
)
|
||||||
from vllm.multimodal.parse import MultiModalDataItems
|
from vllm.multimodal.parse import MultiModalDataItems, VideoProcessorItems
|
||||||
from vllm.multimodal.processing.processor import (
|
from vllm.multimodal.processing.processor import (
|
||||||
MultiModalPromptUpdates,
|
MultiModalPromptUpdates,
|
||||||
PlaceholderFeaturesInfo,
|
PlaceholderFeaturesInfo,
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
|
from vllm.utils.collection_utils import is_list_of
|
||||||
|
|
||||||
from vllm_omni.quantization.component_config import (
|
from vllm_omni.quantization.component_config import (
|
||||||
resolve_encoder_quant_config,
|
resolve_encoder_quant_config,
|
||||||
@@ -74,11 +81,143 @@ except (ImportError, ModuleNotFoundError):
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _presampled_videos_hf_kwargs(
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
mm_kwargs: Mapping[str, object],
|
||||||
|
) -> Mapping[str, object]:
|
||||||
|
"""Adjust HF video kwargs for videos pre-sampled by vLLM's video loader.
|
||||||
|
|
||||||
|
When ``video_metadata`` is present (emitted by
|
||||||
|
``Qwen2_5OmniVideoProcessorItems``), the frames were already sampled
|
||||||
|
according to ``media_io_kwargs``, so the HF processor must not re-sample
|
||||||
|
them, and it needs the sampled fps (instead of its default) to compute
|
||||||
|
``video_second_per_grid`` correctly. Otherwise the audio/video temporal
|
||||||
|
alignment breaks under ``use_audio_in_video=True``.
|
||||||
|
"""
|
||||||
|
video_metadata = mm_data.get("video_metadata")
|
||||||
|
if not video_metadata:
|
||||||
|
return mm_kwargs
|
||||||
|
|
||||||
|
mm_kwargs = dict(mm_kwargs)
|
||||||
|
videos_kwargs = dict(mm_kwargs.get("videos_kwargs") or {})
|
||||||
|
videos_kwargs["do_sample_frames"] = False
|
||||||
|
|
||||||
|
def _compute_sampled_video_fps(metadata) -> float | None:
|
||||||
|
duration = getattr(metadata, "duration", None)
|
||||||
|
indices = getattr(metadata, "frames_indices", None)
|
||||||
|
if not duration or not indices or duration <= 0:
|
||||||
|
return None
|
||||||
|
return len(indices) / float(duration)
|
||||||
|
|
||||||
|
# An explicit user-provided fps takes precedence.
|
||||||
|
if "fps" not in videos_kwargs:
|
||||||
|
fps_values = [_compute_sampled_video_fps(m) for m in video_metadata]
|
||||||
|
# HF accepts a single fps per call and uses it for every video's
|
||||||
|
# video_second_per_grid.
|
||||||
|
known_fps = [fps for fps in fps_values if fps is not None]
|
||||||
|
unique_fps = set(known_fps)
|
||||||
|
if len(unique_fps) == 1:
|
||||||
|
videos_kwargs["fps"] = known_fps[0]
|
||||||
|
elif len(unique_fps) > 1:
|
||||||
|
logger.warning(
|
||||||
|
f"Mixed sampled FPS {sorted(unique_fps)} in one request; HF accepts a single fps, using {known_fps[0]}."
|
||||||
|
)
|
||||||
|
videos_kwargs["fps"] = known_fps[0]
|
||||||
|
|
||||||
|
mm_kwargs["videos_kwargs"] = videos_kwargs
|
||||||
|
return mm_kwargs
|
||||||
|
|
||||||
|
|
||||||
|
class Qwen2_5OmniVideoProcessorItems(VideoProcessorItems):
|
||||||
|
"""Video items that carry the loader's ``(frames, metadata)`` tuples.
|
||||||
|
|
||||||
|
The tuples are kept in ``data`` so that mm hashing and cache-miss
|
||||||
|
re-parsing retain the metadata; ``get_processor_data`` unpacks them into
|
||||||
|
the ``videos`` + ``video_metadata`` arguments of the HF processor.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def get_processor_data(self) -> Mapping[str, object]:
|
||||||
|
from transformers.video_utils import VideoMetadata
|
||||||
|
|
||||||
|
videos = [frames for frames, _ in self.data]
|
||||||
|
video_metadata = [
|
||||||
|
VideoMetadata(**{k: v for k, v in metadata.items() if k != "do_sample_frames"}) for _, metadata in self.data
|
||||||
|
]
|
||||||
|
return {"videos": videos, "video_metadata": video_metadata}
|
||||||
|
|
||||||
|
|
||||||
|
class Qwen2_5OmniThinkerMultiModalDataParser(_Qwen2_5OmniThinkerMultiModalDataParserBase):
|
||||||
|
def _parse_video_data(self, data):
|
||||||
|
if data is None or isinstance(data, dict) or self.is_embeddings(data):
|
||||||
|
return super()._parse_video_data(data)
|
||||||
|
|
||||||
|
# Normalize to a list of per-video items (same as the base parser).
|
||||||
|
if (is_list_of(data, PILImage.Image) and len(data) > 0) or (
|
||||||
|
isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 4
|
||||||
|
):
|
||||||
|
data_items = [data]
|
||||||
|
elif isinstance(data, (np.ndarray, torch.Tensor)):
|
||||||
|
data_items = [elem for elem in data]
|
||||||
|
elif isinstance(data, tuple) and len(data) == 2:
|
||||||
|
data_items = [data]
|
||||||
|
else:
|
||||||
|
data_items = data
|
||||||
|
|
||||||
|
videos_with_metadata = [self._get_video_with_metadata(item) for item in data_items]
|
||||||
|
|
||||||
|
# Metadata is optional: videos fetched by vLLM's media connector
|
||||||
|
# arrive as (frames, metadata) tuples, while plain arrays (e.g.
|
||||||
|
# offline `multi_modal_data` or dummy profiling inputs) carry no
|
||||||
|
# metadata and parse as before.
|
||||||
|
if any(metadata is None for _, metadata in videos_with_metadata):
|
||||||
|
return super()._parse_video_data(data)
|
||||||
|
|
||||||
|
return Qwen2_5OmniVideoProcessorItems(
|
||||||
|
videos_with_metadata,
|
||||||
|
metadata=[metadata for _, metadata in videos_with_metadata],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Qwen2_5OmniThinkerProcessingInfo(_Qwen2_5OmniThinkerProcessingInfoBase):
|
||||||
|
def get_data_parser(self):
|
||||||
|
feature_extractor = self.get_feature_extractor()
|
||||||
|
|
||||||
|
return Qwen2_5OmniThinkerMultiModalDataParser(
|
||||||
|
spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size,
|
||||||
|
target_sr=feature_extractor.sampling_rate,
|
||||||
|
target_channels=self.get_target_channels(),
|
||||||
|
expected_hidden_size=self._get_expected_hidden_size(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Qwen2_5OmniThinkerMultiModalProcessor(
|
class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||||
_Qwen2_5OmniThinkerMultiModalProcessorBase,
|
_Qwen2_5OmniThinkerMultiModalProcessorBase,
|
||||||
):
|
):
|
||||||
"""Override to fix use_audio_in_video detection when mm cache returns None."""
|
"""Override to fix use_audio_in_video detection when mm cache returns None."""
|
||||||
|
|
||||||
|
def _cached_apply_hf_processor(self, inputs, timing_ctx):
|
||||||
|
# If use_audio_in_video, process video and audio together; otherwise,
|
||||||
|
# skip cache to avoid errors.
|
||||||
|
# Prevents partial cache hits from causing processing failures.
|
||||||
|
if inputs.hf_processor_mm_kwargs.get("use_audio_in_video"):
|
||||||
|
return self._apply_hf_processor(inputs, timing_ctx)
|
||||||
|
return super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||||
|
|
||||||
|
def _call_hf_processor(
|
||||||
|
self,
|
||||||
|
prompt: str,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
mm_kwargs: Mapping[str, object],
|
||||||
|
tok_kwargs: Mapping[str, object],
|
||||||
|
):
|
||||||
|
mm_kwargs = _presampled_videos_hf_kwargs(mm_data, mm_kwargs)
|
||||||
|
return super()._call_hf_processor(
|
||||||
|
prompt=prompt,
|
||||||
|
mm_data=mm_data,
|
||||||
|
mm_kwargs=mm_kwargs,
|
||||||
|
tok_kwargs=tok_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
def _maybe_apply_prompt_updates(
|
def _maybe_apply_prompt_updates(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
|
|||||||
@@ -74,7 +74,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
|||||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||||
Qwen2_5OmniAudioFeatureInputs,
|
Qwen2_5OmniAudioFeatureInputs,
|
||||||
Qwen2_5OmniThinkerDummyInputsBuilder,
|
Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
|
||||||
check_interleaved_audio_video,
|
check_interleaved_audio_video,
|
||||||
merge_interleaved_embeddings,
|
merge_interleaved_embeddings,
|
||||||
)
|
)
|
||||||
@@ -120,6 +119,9 @@ from vllm.transformers_utils.processor import cached_processor_from_config
|
|||||||
|
|
||||||
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
|
from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
|
||||||
Qwen2_5OmniConditionalGenerationMixin,
|
Qwen2_5OmniConditionalGenerationMixin,
|
||||||
|
Qwen2_5OmniThinkerMultiModalDataParser,
|
||||||
|
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||||
|
_presampled_videos_hf_kwargs,
|
||||||
)
|
)
|
||||||
from vllm_omni.quantization.component_config import (
|
from vllm_omni.quantization.component_config import (
|
||||||
PRE_QUANTIZED_METHODS,
|
PRE_QUANTIZED_METHODS,
|
||||||
@@ -621,6 +623,19 @@ class Qwen3OmniMoeThinkerProcessingInfo(Qwen2AudioProcessingInfo, Qwen2_5_VLProc
|
|||||||
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
||||||
return feature_extractor
|
return feature_extractor
|
||||||
|
|
||||||
|
def get_data_parser(self):
|
||||||
|
feature_extractor = self.get_feature_extractor()
|
||||||
|
|
||||||
|
# Install the Omni parser, which keeps video metadata from vLLM's
|
||||||
|
# video loader (see Qwen2_5OmniVideoProcessorItems); the parser
|
||||||
|
# inherited from Qwen2AudioProcessingInfo would drop it.
|
||||||
|
return Qwen2_5OmniThinkerMultiModalDataParser(
|
||||||
|
spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size,
|
||||||
|
target_sr=feature_extractor.sampling_rate,
|
||||||
|
target_channels=1,
|
||||||
|
expected_hidden_size=self._get_expected_hidden_size(),
|
||||||
|
)
|
||||||
|
|
||||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||||
return {"audio": None, "image": None, "video": None}
|
return {"audio": None, "image": None, "video": None}
|
||||||
|
|
||||||
@@ -671,8 +686,11 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
|||||||
tok_kwargs: Mapping[str, object],
|
tok_kwargs: Mapping[str, object],
|
||||||
) -> BatchFeature:
|
) -> BatchFeature:
|
||||||
mm_data = dict(mm_data)
|
mm_data = dict(mm_data)
|
||||||
|
mm_kwargs = dict(mm_kwargs)
|
||||||
audios = mm_data.pop("audios", [])
|
audios = mm_data.pop("audios", [])
|
||||||
|
|
||||||
|
mm_kwargs = dict(_presampled_videos_hf_kwargs(mm_data, mm_kwargs))
|
||||||
|
|
||||||
def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray:
|
def pad_to_hop_length(x: np.ndarray, hop_length: int) -> np.ndarray:
|
||||||
length = x.shape[-1]
|
length = x.shape[-1]
|
||||||
if length % hop_length != 0:
|
if length % hop_length != 0:
|
||||||
@@ -927,9 +945,13 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
|||||||
|
|
||||||
audio_in_video_item_idx += 1
|
audio_in_video_item_idx += 1
|
||||||
|
|
||||||
second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
|
# Prefer the HF-computed value (temporal_patch_size / sampled fps)
|
||||||
if second_per_grid_ts:
|
# over request kwargs;
|
||||||
video_second_per_grid_t = second_per_grid_ts[item_idx]
|
second_per_grid_ts = out_mm_data.get("second_per_grid_ts")
|
||||||
|
if second_per_grid_ts is None:
|
||||||
|
second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
|
||||||
|
if second_per_grid_ts is not None:
|
||||||
|
video_second_per_grid_t = float(second_per_grid_ts[item_idx])
|
||||||
else:
|
else:
|
||||||
video_second_per_grid_t = 2.0
|
video_second_per_grid_t = 2.0
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user