[Refactor]Refactoring audio_in_video implementation (#3566)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-06-08 14:14:02 +08:00
1 parent 5f1a876cbd
commit fc7393e485
4 files changed
+206 -349

No files matched your search

@@ -1,8 +1,9 @@
"""Thin Omni wrapper: reuse upstream Qwen2.5-Omni thinker (v0.14) with minimal overrides."""
"""Thin Omni wrapper: reuse upstream Qwen2.5-Omni thinker with minimal overrides."""
from collections.abc import Iterable, Mapping
from collections.abc import Iterable, Iterator, Mapping
from typing import Any
import numpy as np
import torch
from torch import nn
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
@@ -49,9 +50,7 @@ from vllm.model_executor.models.utils import (
WeightsMapper,
init_vllm_registered_model,
maybe_prefix,
split_list_into_ranges,
)
from vllm.model_executor.models.vision import get_llm_pos_ids_for_vision
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -92,20 +91,15 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
# Detect use_audio_in_video from mm_kwargs
use_audio_in_video = False
if "video" in mm_kwargs:
non_none_items = [item for item in mm_kwargs["video"] if item is not None]
if non_none_items:
for item in non_none_items:
if item.get("use_audio_in_video"):
uaiv_tensor = item["use_audio_in_video"].data
if uaiv_tensor.numel() > 0:
use_audio_in_video = bool(uaiv_tensor.item())
break
elif "audio" in mm_prompt_updates:
tokenizer = self.info.get_tokenizer()
audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
use_audio_in_video = audio_pad_id not in prompt_ids
for item in mm_kwargs["video"]:
if item and item.get("use_audio_in_video"):
use_audio_in_video_tensor = item["use_audio_in_video"].data
if use_audio_in_video_tensor.numel() > 0:
use_audio_in_video = bool(use_audio_in_video_tensor.item())
break
# for mutilmodality cache
if any(item is None for item in mm_kwargs["video"]):
video_token_id = self.info.get_hf_config().video_token_id
@@ -421,167 +415,193 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
def get_language_model(self) -> torch.nn.Module:
return self.language_model
def _get_audio_for_video_mapping(self, mm_features: list[MultiModalFeatureSpec]) -> tuple[dict[int, int], set[int]]:
"""
Map video offset -> paired audio_feature_length for use_audio_in_video.
When use_audio_in_video=True, audio is interleaved within video chunks.
The pairing is based on feature order in mm_features.
"""
videos_with_audio = [
f
for f in mm_features
if f.modality == "video" and f.data.get("use_audio_in_video") and f.data["use_audio_in_video"].data.item()
]
audios = [f for f in mm_features if f.modality == "audio"]
mapping: dict[int, int] = {}
paired_audio_offsets: set[int] = set()
for i, video_f in enumerate(videos_with_audio):
if i < len(audios):
audio_len = audios[i].data["audio_feature_lengths"].data.item()
mapping[video_f.mm_position.offset] = audio_len
paired_audio_offsets.add(audios[i].mm_position.offset)
return mapping, paired_audio_offsets
def _compute_audio_token_count(self, audio_feature_length: int) -> int:
return ((audio_feature_length - 1) // 2 + 1 - 2) // 2 + 1
def iter_mm_features(self, mm_features: list[MultiModalFeatureSpec]) -> Iterator[tuple[int, str, dict[str, Any]]]:
thinker_config = self.config
spatial_merge_size = thinker_config.vision_config.spatial_merge_size
tokens_per_second = getattr(thinker_config.vision_config, "tokens_per_second", 25)
sorted_features = sorted(mm_features, key=lambda f: f.mm_position.offset)
audio_for_video, paired_audio_offsets = self._get_audio_for_video_mapping(sorted_features)
for mm_feature in sorted_features:
offset = mm_feature.mm_position.offset
modality = mm_feature.modality
if modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
yield (
offset,
"image",
{
"grid_t": t,
"grid_h": h // spatial_merge_size,
"grid_w": w // spatial_merge_size,
"t_factor": 1.0 * tokens_per_second,
},
)
elif modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
second_per_grid_ts = 1.0
if mm_feature.data.get("second_per_grid_ts"):
second_per_grid_ts = mm_feature.data["second_per_grid_ts"].data.item()
use_audio_in_video = False
if mm_feature.data.get("use_audio_in_video"):
use_audio_in_video = bool(mm_feature.data["use_audio_in_video"].data.item())
yield (
offset,
"video",
{
"grid_t": t,
"grid_h": h // spatial_merge_size,
"grid_w": w // spatial_merge_size,
"t_factor": second_per_grid_ts * tokens_per_second,
"use_audio_in_video": use_audio_in_video,
"audio_feature_length": audio_for_video.get(offset),
},
)
elif modality == "audio":
if offset not in paired_audio_offsets:
audio_len = mm_feature.data["audio_feature_lengths"].data.item()
yield offset, "audio", {"audio_feature_length": audio_len}
def _compute_interleaved_positions(self, start_idx: int, data: dict[str, Any]) -> tuple[np.ndarray, int]:
grid_t = data["grid_t"]
grid_h = data["grid_h"]
grid_w = data["grid_w"]
t_factor = data["t_factor"]
audio_len = data["audio_feature_length"]
thinker_config = self.config
tokens_per_second = getattr(thinker_config.vision_config, "tokens_per_second", 25)
seconds_per_chunk = thinker_config.seconds_per_chunk
t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
t_index = (np.arange(grid_t) * t_factor).astype(np.int64)
t_index_split_chunk: list[list[int]] = [[] for _ in range((int(t_index.max()) // t_ntoken_per_chunk) + 1)]
for t_val in t_index:
idx = int(t_val) // t_ntoken_per_chunk
t_index_split_chunk[idx].append(int(t_val))
pure_audio_len = self._compute_audio_token_count(audio_len)
added_audio_len = 0
pos_ids_list: list[np.ndarray] = []
audio_start_idx = start_idx
for t_chunk in t_index_split_chunk:
if not t_chunk:
continue
chunk_t = len(t_chunk)
h_indices = np.tile(np.arange(grid_h).reshape(1, -1, 1), (chunk_t, 1, grid_w)).flatten()
w_indices = np.tile(np.arange(grid_w).reshape(1, 1, -1), (chunk_t, grid_h, 1)).flatten()
t_indices = np.repeat(np.array(t_chunk), grid_h * grid_w)
vision_pos = np.stack([t_indices, h_indices, w_indices]) + start_idx
pos_ids_list.append(vision_pos)
audio_chunk_size = min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
if audio_chunk_size > 0:
audio_pos = np.broadcast_to(np.arange(audio_chunk_size), (3, audio_chunk_size)) + audio_start_idx
pos_ids_list.append(audio_pos)
audio_start_idx = audio_start_idx + audio_chunk_size
added_audio_len += audio_chunk_size
if added_audio_len < pure_audio_len:
remaining = pure_audio_len - added_audio_len
remaining_audio_pos = np.broadcast_to(np.arange(remaining), (3, remaining)) + audio_start_idx
pos_ids_list.append(remaining_audio_pos)
vision_tokens = grid_t * grid_h * grid_w
total_tokens = vision_tokens + pure_audio_len
return np.concatenate(pos_ids_list, axis=1), total_tokens
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
"""
Example:
llm_pos_ids_list: list[np.ndarray] = []
st = 0
(V_i are vision position ids, A_i are audio position ids)
for offset, modality, data in self.iter_mm_features(mm_features):
text_len = offset - st
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
if text_len > 0:
llm_pos_ids_list.append(np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx)
st_idx += text_len
|V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
|vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
"""
kwargs = MultiModalFeatureSpec.gather_kwargs(
mm_features,
{
"image_grid_thw",
"video_grid_thw",
"second_per_grid_ts",
"audio_feature_lengths",
"use_audio_in_video",
},
)
image_grid_thw = kwargs.get("image_grid_thw", [])
video_grid_thw = kwargs.get("video_grid_thw", [])
second_per_grid_ts = kwargs.get("second_per_grid_ts", [])
audio_feature_lengths = kwargs.get("audio_feature_lengths", [])
use_audio_in_video = any(kwargs.get("use_audio_in_video", []))
if modality == "audio":
audio_tokens = self._compute_audio_token_count(data["audio_feature_length"])
llm_pos_ids_list.append(np.broadcast_to(np.arange(audio_tokens), (3, audio_tokens)) + st_idx)
st = offset + audio_tokens
image_grid_thw = (torch.stack if image_grid_thw else torch.tensor)(image_grid_thw)
video_grid_thw = (torch.stack if video_grid_thw else torch.tensor)(video_grid_thw)
elif modality == "image":
grid_t = data["grid_t"]
grid_h = data["grid_h"]
grid_w = data["grid_w"]
t_factor = data["t_factor"]
# TODO(fyabc): refactor and share more code with
# _vl_get_input_positions_tensor.
grid_indices = np.indices((grid_t, grid_h, grid_w))
if t_factor != 1.0:
grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
st = offset + grid_t * grid_h * grid_w
thinker_config = self.config
audio_token_id = thinker_config.audio_token_index
image_token_id = thinker_config.image_token_index
video_token_id = thinker_config.video_token_index
audio_start_token_id = thinker_config.audio_start_token_id
audio_end_token_id = thinker_config.audio_end_token_id
vision_start_token_id = thinker_config.vision_start_token_id
vision_end_token_id = thinker_config.vision_end_token_id
seconds_per_chunk = thinker_config.seconds_per_chunk
spatial_merge_size = thinker_config.vision_config.spatial_merge_size
tokens_per_second = getattr(thinker_config.vision_config, "tokens_per_second", 25)
elif modality == "video":
grid_t = data["grid_t"]
grid_h = data["grid_h"]
grid_w = data["grid_w"]
t_factor = data["t_factor"]
src_item = input_tokens
audio_seqlens = audio_feature_lengths
if not second_per_grid_ts:
second_per_grid_ts = [1] * video_grid_thw.shape[0]
audio_idx = 0
video_idx = 0
image_idx = 0
new_src_item: list[int] = []
llm_pos_ids_list: list[torch.Tensor] = []
if not data["use_audio_in_video"]:
grid_indices = np.indices((grid_t, grid_h, grid_w))
if t_factor != 1.0:
grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
st = offset + grid_t * grid_h * grid_w
else:
pos_ids, token_count = self._compute_interleaved_positions(st_idx, data)
llm_pos_ids_list.append(pos_ids)
st = offset + token_count
idx = 0
while idx < len(src_item):
new_src_item_len = len(new_src_item)
start_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
if src_item[idx] not in [audio_token_id, video_token_id, image_token_id]:
if use_audio_in_video and idx > 0:
if src_item[idx] == vision_end_token_id and src_item[idx - 1] == audio_end_token_id:
# processing the <|audio_eos|> before <|vision_eos|>
start_idx -= 1
elif src_item[idx] == audio_start_token_id and src_item[idx - 1] == vision_start_token_id:
# processing the <|audio_bos|> after <|vision_eos|>
start_idx -= 1
new_src_item.append(src_item[idx])
llm_pos_ids = torch.tensor([start_idx], dtype=torch.long).expand(3, -1)
llm_pos_ids_list.append(llm_pos_ids)
elif src_item[idx] == audio_token_id:
assert audio_seqlens is not None
audio_seqlen = audio_seqlens[audio_idx]
place_num = ((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1
new_src_item.extend([audio_token_id] * place_num)
llm_pos_ids = torch.arange(place_num).expand(3, -1) + start_idx
llm_pos_ids_list.append(llm_pos_ids)
audio_idx += 1
elif src_item[idx] == image_token_id:
grid_t = image_grid_thw[image_idx][0]
grid_hs = image_grid_thw[:, 1]
grid_ws = image_grid_thw[:, 2]
t_index = (torch.arange(grid_t) * 1 * tokens_per_second).long()
llm_pos_ids = get_llm_pos_ids_for_vision(
start_idx, image_idx, spatial_merge_size, t_index, grid_hs, grid_ws
)
llm_pos_ids_list.append(llm_pos_ids)
vision_seqlen = image_grid_thw[image_idx].prod() // (spatial_merge_size**2)
new_src_item.extend([image_token_id] * vision_seqlen)
image_idx += 1
elif src_item[idx] == video_token_id and not use_audio_in_video:
grid_t = video_grid_thw[video_idx][0]
grid_hs = video_grid_thw[:, 1]
grid_ws = video_grid_thw[:, 2]
t_index = (torch.arange(grid_t) * second_per_grid_ts[video_idx] * tokens_per_second).long()
llm_pos_ids = get_llm_pos_ids_for_vision(
start_idx, video_idx, spatial_merge_size, t_index, grid_hs, grid_ws
)
llm_pos_ids_list.append(llm_pos_ids)
vision_seqlen = video_grid_thw[video_idx].prod() // (spatial_merge_size**2)
new_src_item.extend([video_token_id] * vision_seqlen)
video_idx += 1
else:
# read audio from video
assert audio_seqlens is not None
audio_seqlen = audio_seqlens[audio_idx]
vision_seqlen = video_grid_thw[video_idx].prod() // (spatial_merge_size**2)
grid_t = video_grid_thw[video_idx][0]
grid_h = video_grid_thw[video_idx][1]
grid_w = video_grid_thw[video_idx][2]
grid_hs = video_grid_thw[:, 1]
grid_ws = video_grid_thw[:, 2]
t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
t_index = (torch.arange(grid_t) * second_per_grid_ts[video_idx] * tokens_per_second).long()
t_index_split_chunk = split_list_into_ranges(t_index, t_ntoken_per_chunk)
place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1) + 2
pure_audio_len = place_num - 2
added_audio_len = 0
audio_llm_pos_ids_list: list[torch.Tensor] = []
for t_chunk in t_index_split_chunk:
vision_ntoken_per_chunk = len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
new_src_item.extend([video_token_id] * vision_ntoken_per_chunk)
vision_llm_pos_ids_list = get_llm_pos_ids_for_vision(
start_idx,
video_idx,
spatial_merge_size,
t_chunk,
grid_hs,
grid_ws,
).split(1, dim=1)
llm_pos_ids_list.extend(vision_llm_pos_ids_list)
new_src_item.extend(min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) * [audio_token_id])
audio_start_idx = (
start_idx if len(audio_llm_pos_ids_list) == 0 else audio_llm_pos_ids_list[-1][0].item() + 1
)
if min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) > 0:
audio_llm_pos_ids_list = (
torch.arange(min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)).expand(3, -1)
+ audio_start_idx
).split(1, dim=1)
else:
audio_llm_pos_ids_list = []
added_audio_len += min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
llm_pos_ids_list.extend(audio_llm_pos_ids_list)
if added_audio_len < pure_audio_len:
new_src_item.extend((pure_audio_len - added_audio_len) * [audio_token_id])
audio_llm_pos_ids_list = (
torch.arange(pure_audio_len - added_audio_len).expand(3, -1) + llm_pos_ids_list[-1].max() + 1
).split(1, dim=1)
llm_pos_ids_list.extend(audio_llm_pos_ids_list)
audio_idx += 1
video_idx += 1
# move to the next token
idx += len(new_src_item) - new_src_item_len
if st < len(input_tokens):
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
text_len = len(input_tokens) - st
llm_pos_ids_list.append(np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx)
llm_positions = torch.cat(llm_pos_ids_list, dim=1)
mrope_position_delta = torch.cat(llm_pos_ids_list, dim=1).max() + 1 - len(src_item)
llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
mrope_position_delta = int(llm_positions.max()) + 1 - len(input_tokens)
return llm_positions, mrope_position_delta
return torch.from_numpy(llm_positions), mrope_position_delta
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
@@ -629,14 +649,12 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
# Check for audio-in-video: interleaved video and audio tokens
# in the multimodal region. Only use the interleaved path when
# needed; otherwise fall back to the default parent implementation.
# vLLM's _gather_mm_embeddings builds is_multimodal on CPU for merge
# indexing; bitwise ops with input_ids require the same device.
is_mm_device = is_multimodal.to(device=input_ids.device, non_blocking=True)
video_token_id = self.config.video_token_index
audio_token_id = self.config.audio_token_index
is_video = is_mm_device & (input_ids == video_token_id)
is_audio = is_mm_device & (input_ids == audio_token_id)
input_ids_cpu = input_ids.cpu()
is_video = is_multimodal & (input_ids_cpu == video_token_id)
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
num_video = is_video.sum().item()
num_audio = is_audio.sum().item()
@@ -652,7 +670,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
multimodal_embeddings,
is_video,
is_audio,
is_mm_device,
is_multimodal,
num_video,
num_audio,
)
@@ -758,17 +758,11 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
use_audio_in_video = False
if "video" in mm_kwargs:
non_none_items = [item for item in mm_kwargs["video"] if item is not None]
if non_none_items:
# Normal case: at least one non-cached item, read flag directly
use_audio_in_video = any(item["use_audio_in_video"].data for item in non_none_items)
elif "audio" in mm_prompt_updates:
# All video items are from cache (None); infer from prompt:
# use_audio_in_video=True means the prompt has no <|audio_pad|>
# placeholder (audio is embedded in video tokens instead)
tokenizer = self.info.get_tokenizer()
audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>")
use_audio_in_video = audio_pad_id not in prompt_ids
for item in mm_kwargs["video"]:
if item and item["use_audio_in_video"].data:
use_audio_in_video = True
else:
use_audio_in_video = False
# for mutilmodality cache
if any(item is None for item in mm_kwargs["video"]):
video_token_id = self.info.get_hf_config().video_token_id