diff --git a/vllm_omni/entrypoints/chat_utils.py b/vllm_omni/entrypoints/chat_utils.py deleted file mode 100644 index 4c3d311ec..000000000 --- a/vllm_omni/entrypoints/chat_utils.py +++ /dev/null @@ -1,62 +0,0 @@ -import numpy as np - - -async def extract_audio_from_video_async(video_url: str) -> tuple[np.ndarray, int | float]: - """Extract audio from a video URL using vllm's load_audio. - - Returns a (audio_array, sample_rate) tuple compatible with audio format. - All blocking I/O operations are run in a thread pool. - """ - import asyncio - import os - import tempfile - from urllib.parse import urlparse - - parsed_url = urlparse(video_url) - temp_video_file_path = None - - def _download_video_sync(url: str) -> bytes: - from urllib.request import urlopen - - return urlopen(url).read() - - def _write_temp_file_sync(data: bytes, suffix: str) -> str: - with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file: - temp_file.write(data) - return temp_file.name - - def _load_audio_sync(file_path: str) -> tuple[np.ndarray, int | float]: - from vllm.multimodal.media.audio import load_audio - - return load_audio(file_path, sr=16000) - - def _cleanup_file_sync(file_path: str) -> None: - try: - if os.path.exists(file_path): - os.unlink(file_path) - except OSError: - pass - - try: - if parsed_url.scheme in ("http", "https"): - video_data = await asyncio.to_thread(_download_video_sync, video_url) - temp_video_file_path = await asyncio.to_thread(_write_temp_file_sync, video_data, ".mp4") - elif parsed_url.scheme == "file": - from urllib.request import url2pathname - - temp_video_file_path = url2pathname(parsed_url.path) - elif parsed_url.scheme == "data": - import base64 - - header, data = video_url.split(",", 1) - video_data = base64.b64decode(data) - temp_video_file_path = await asyncio.to_thread(_write_temp_file_sync, video_data, ".mp4") - else: - # Assume local file path - temp_video_file_path = video_url - - audio_array, sample_rate = await asyncio.to_thread(_load_audio_sync, temp_video_file_path) - return audio_array, sample_rate - finally: - if temp_video_file_path and parsed_url.scheme in ("http", "https", "data"): - await asyncio.to_thread(_cleanup_file_sync, temp_video_file_path) diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 9bac6c9ba..5dec1768a 100755 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -12,8 +12,16 @@ from typing import Any, Final, cast import jinja2 import torch from fastapi import Request +from openai.types.chat.chat_completion_audio import ChatCompletionAudio as OpenAIChatCompletionAudio from PIL import Image from pydantic import TypeAdapter +from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ChatTemplateContentFormatOption, + ConversationMessage, + get_history_tool_calls_cnt, + make_tool_call_id, +) from vllm_omni.diffusion.diffusion_engine import get_extra_body_params, get_extra_output_params from vllm_omni.entrypoints.async_omni import AsyncOmni @@ -32,14 +40,6 @@ except ImportError: soundfile = None -from openai.types.chat.chat_completion_audio import ChatCompletionAudio as OpenAIChatCompletionAudio -from vllm.entrypoints.chat_utils import ( - ChatCompletionMessageParam, - ChatTemplateContentFormatOption, - ConversationMessage, - get_history_tool_calls_cnt, - make_tool_call_id, -) from vllm.entrypoints.launcher import terminate_if_errored from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, @@ -652,19 +652,15 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): ) tok_params = request.build_tok_params(self.model_config) + mm_config = self.model_config.multimodal_config chat_params = request.build_chat_params( default_template, default_template_content_format, - ).with_defaults(default_template_kwargs) - - # OMNI: When use_audio_in_video=True, the qwen2_5_omni_thinker mm - # processor asserts that audio items are present alongside video items - # (inside render_chat_async). We must inject audio_url items into the - # messages BEFORE calling render_chat_async so the mm processor can - # count them correctly during tokenisation. - mm_proc_kw = getattr(request, "mm_processor_kwargs", None) or {} - if mm_proc_kw.get("use_audio_in_video", False): - messages = await self._inject_audio_from_video_urls(messages) + ).with_defaults( + default_template_kwargs, + default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), + default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), + ) (conversation,), (engine_prompt,) = await renderer.render_chat_async( [messages], @@ -735,95 +731,6 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): return conversation, [engine_prompt] - async def _inject_audio_from_video_urls( - self, - messages: list[ChatCompletionMessageParam], - ) -> list[ChatCompletionMessageParam]: - """Pre-extract audio from video URLs and inject as audio_url content items. - - When use_audio_in_video=True, the qwen2_5_omni_thinker multimodal - processor requires that the number of audio items equals the number of - video items (it subtracts mm_counts["video"] from mm_counts["audio"]). - The client only sends video_url items; this method adds the matching - audio_url items on the server side before the renderer processes them. - """ - import io - - from vllm_omni.entrypoints.chat_utils import extract_audio_from_video_async - - new_messages: list[ChatCompletionMessageParam] = [] - for msg in messages: - content = msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None) - if not isinstance(content, list): - new_messages.append(msg) - continue - - video_urls = [ - part.get("video_url", {}).get("url") - for part in content - if isinstance(part, dict) and part.get("type") == "video_url" and part.get("video_url", {}).get("url") - ] - - if not video_urls: - new_messages.append(msg) - continue - - audios = await asyncio.gather(*(extract_audio_from_video_async(u) for u in video_urls)) - - audio_items: list[dict] = [] - for audio_array, sample_rate in audios: - buf = io.BytesIO() - if soundfile is not None: - soundfile.write(buf, audio_array, samplerate=int(sample_rate), format="WAV") - else: - import struct - - import numpy as np - - audio_np = np.asarray(audio_array, dtype=np.float32) - sr = int(sample_rate) - num_channels = 1 - bits_per_sample = 32 - num_frames = len(audio_np) - data_size = num_frames * num_channels * (bits_per_sample // 8) - # Write minimal RIFF/WAV header - buf.write(b"RIFF") - buf.write(struct.pack(" list[Any]: """Convert request dicts to stage-typed sampling params objects. diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py index 931049d8d..5d58a63d4 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py @@ -1,8 +1,9 @@ -"""Thin Omni wrapper: reuse upstream Qwen2.5-Omni thinker (v0.14) with minimal overrides.""" +"""Thin Omni wrapper: reuse upstream Qwen2.5-Omni thinker with minimal overrides.""" -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import Any +import numpy as np import torch from torch import nn from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import ( @@ -49,9 +50,7 @@ from vllm.model_executor.models.utils import ( WeightsMapper, init_vllm_registered_model, maybe_prefix, - split_list_into_ranges, ) -from vllm.model_executor.models.vision import get_llm_pos_ids_for_vision from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFeatureSpec, @@ -92,20 +91,15 @@ class Qwen2_5OmniThinkerMultiModalProcessor( self._validate_mm_kwargs(mm_kwargs, mm_item_counts) self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + # Detect use_audio_in_video from mm_kwargs use_audio_in_video = False if "video" in mm_kwargs: - non_none_items = [item for item in mm_kwargs["video"] if item is not None] - if non_none_items: - for item in non_none_items: - if item.get("use_audio_in_video"): - uaiv_tensor = item["use_audio_in_video"].data - if uaiv_tensor.numel() > 0: - use_audio_in_video = bool(uaiv_tensor.item()) - break - elif "audio" in mm_prompt_updates: - tokenizer = self.info.get_tokenizer() - audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>") - use_audio_in_video = audio_pad_id not in prompt_ids + for item in mm_kwargs["video"]: + if item and item.get("use_audio_in_video"): + use_audio_in_video_tensor = item["use_audio_in_video"].data + if use_audio_in_video_tensor.numel() > 0: + use_audio_in_video = bool(use_audio_in_video_tensor.item()) + break # for mutilmodality cache if any(item is None for item in mm_kwargs["video"]): video_token_id = self.info.get_hf_config().video_token_id @@ -421,167 +415,193 @@ class Qwen2_5OmniThinkerForConditionalGeneration( def get_language_model(self) -> torch.nn.Module: return self.language_model + def _get_audio_for_video_mapping(self, mm_features: list[MultiModalFeatureSpec]) -> tuple[dict[int, int], set[int]]: + """ + Map video offset -> paired audio_feature_length for use_audio_in_video. + + When use_audio_in_video=True, audio is interleaved within video chunks. + The pairing is based on feature order in mm_features. + """ + videos_with_audio = [ + f + for f in mm_features + if f.modality == "video" and f.data.get("use_audio_in_video") and f.data["use_audio_in_video"].data.item() + ] + audios = [f for f in mm_features if f.modality == "audio"] + + mapping: dict[int, int] = {} + paired_audio_offsets: set[int] = set() + for i, video_f in enumerate(videos_with_audio): + if i < len(audios): + audio_len = audios[i].data["audio_feature_lengths"].data.item() + mapping[video_f.mm_position.offset] = audio_len + paired_audio_offsets.add(audios[i].mm_position.offset) + return mapping, paired_audio_offsets + + def _compute_audio_token_count(self, audio_feature_length: int) -> int: + return ((audio_feature_length - 1) // 2 + 1 - 2) // 2 + 1 + + def iter_mm_features(self, mm_features: list[MultiModalFeatureSpec]) -> Iterator[tuple[int, str, dict[str, Any]]]: + thinker_config = self.config + spatial_merge_size = thinker_config.vision_config.spatial_merge_size + tokens_per_second = getattr(thinker_config.vision_config, "tokens_per_second", 25) + + sorted_features = sorted(mm_features, key=lambda f: f.mm_position.offset) + audio_for_video, paired_audio_offsets = self._get_audio_for_video_mapping(sorted_features) + + for mm_feature in sorted_features: + offset = mm_feature.mm_position.offset + modality = mm_feature.modality + + if modality == "image": + t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + yield ( + offset, + "image", + { + "grid_t": t, + "grid_h": h // spatial_merge_size, + "grid_w": w // spatial_merge_size, + "t_factor": 1.0 * tokens_per_second, + }, + ) + elif modality == "video": + t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + second_per_grid_ts = 1.0 + if mm_feature.data.get("second_per_grid_ts"): + second_per_grid_ts = mm_feature.data["second_per_grid_ts"].data.item() + use_audio_in_video = False + if mm_feature.data.get("use_audio_in_video"): + use_audio_in_video = bool(mm_feature.data["use_audio_in_video"].data.item()) + + yield ( + offset, + "video", + { + "grid_t": t, + "grid_h": h // spatial_merge_size, + "grid_w": w // spatial_merge_size, + "t_factor": second_per_grid_ts * tokens_per_second, + "use_audio_in_video": use_audio_in_video, + "audio_feature_length": audio_for_video.get(offset), + }, + ) + elif modality == "audio": + if offset not in paired_audio_offsets: + audio_len = mm_feature.data["audio_feature_lengths"].data.item() + yield offset, "audio", {"audio_feature_length": audio_len} + + def _compute_interleaved_positions(self, start_idx: int, data: dict[str, Any]) -> tuple[np.ndarray, int]: + grid_t = data["grid_t"] + grid_h = data["grid_h"] + grid_w = data["grid_w"] + t_factor = data["t_factor"] + audio_len = data["audio_feature_length"] + + thinker_config = self.config + tokens_per_second = getattr(thinker_config.vision_config, "tokens_per_second", 25) + seconds_per_chunk = thinker_config.seconds_per_chunk + t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk) + + t_index = (np.arange(grid_t) * t_factor).astype(np.int64) + t_index_split_chunk: list[list[int]] = [[] for _ in range((int(t_index.max()) // t_ntoken_per_chunk) + 1)] + for t_val in t_index: + idx = int(t_val) // t_ntoken_per_chunk + t_index_split_chunk[idx].append(int(t_val)) + + pure_audio_len = self._compute_audio_token_count(audio_len) + added_audio_len = 0 + pos_ids_list: list[np.ndarray] = [] + audio_start_idx = start_idx + + for t_chunk in t_index_split_chunk: + if not t_chunk: + continue + + chunk_t = len(t_chunk) + + h_indices = np.tile(np.arange(grid_h).reshape(1, -1, 1), (chunk_t, 1, grid_w)).flatten() + w_indices = np.tile(np.arange(grid_w).reshape(1, 1, -1), (chunk_t, grid_h, 1)).flatten() + t_indices = np.repeat(np.array(t_chunk), grid_h * grid_w) + + vision_pos = np.stack([t_indices, h_indices, w_indices]) + start_idx + pos_ids_list.append(vision_pos) + + audio_chunk_size = min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) + if audio_chunk_size > 0: + audio_pos = np.broadcast_to(np.arange(audio_chunk_size), (3, audio_chunk_size)) + audio_start_idx + pos_ids_list.append(audio_pos) + audio_start_idx = audio_start_idx + audio_chunk_size + added_audio_len += audio_chunk_size + + if added_audio_len < pure_audio_len: + remaining = pure_audio_len - added_audio_len + remaining_audio_pos = np.broadcast_to(np.arange(remaining), (3, remaining)) + audio_start_idx + pos_ids_list.append(remaining_audio_pos) + + vision_tokens = grid_t * grid_h * grid_w + total_tokens = vision_tokens + pure_audio_len + + return np.concatenate(pos_ids_list, axis=1), total_tokens + def get_mrope_input_positions( self, input_tokens: list[int], mm_features: list[MultiModalFeatureSpec], ) -> tuple[torch.Tensor, int]: - """ - Example: + llm_pos_ids_list: list[np.ndarray] = [] + st = 0 - (V_i are vision position ids, A_i are audio position ids) + for offset, modality, data in self.iter_mm_features(mm_features): + text_len = offset - st + st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0 + if text_len > 0: + llm_pos_ids_list.append(np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx) + st_idx += text_len - |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|... - |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |... - """ - kwargs = MultiModalFeatureSpec.gather_kwargs( - mm_features, - { - "image_grid_thw", - "video_grid_thw", - "second_per_grid_ts", - "audio_feature_lengths", - "use_audio_in_video", - }, - ) - image_grid_thw = kwargs.get("image_grid_thw", []) - video_grid_thw = kwargs.get("video_grid_thw", []) - second_per_grid_ts = kwargs.get("second_per_grid_ts", []) - audio_feature_lengths = kwargs.get("audio_feature_lengths", []) - use_audio_in_video = any(kwargs.get("use_audio_in_video", [])) + if modality == "audio": + audio_tokens = self._compute_audio_token_count(data["audio_feature_length"]) + llm_pos_ids_list.append(np.broadcast_to(np.arange(audio_tokens), (3, audio_tokens)) + st_idx) + st = offset + audio_tokens - image_grid_thw = (torch.stack if image_grid_thw else torch.tensor)(image_grid_thw) - video_grid_thw = (torch.stack if video_grid_thw else torch.tensor)(video_grid_thw) + elif modality == "image": + grid_t = data["grid_t"] + grid_h = data["grid_h"] + grid_w = data["grid_w"] + t_factor = data["t_factor"] - # TODO(fyabc): refactor and share more code with - # _vl_get_input_positions_tensor. + grid_indices = np.indices((grid_t, grid_h, grid_w)) + if t_factor != 1.0: + grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64) + llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx) + st = offset + grid_t * grid_h * grid_w - thinker_config = self.config - audio_token_id = thinker_config.audio_token_index - image_token_id = thinker_config.image_token_index - video_token_id = thinker_config.video_token_index - audio_start_token_id = thinker_config.audio_start_token_id - audio_end_token_id = thinker_config.audio_end_token_id - vision_start_token_id = thinker_config.vision_start_token_id - vision_end_token_id = thinker_config.vision_end_token_id - seconds_per_chunk = thinker_config.seconds_per_chunk - spatial_merge_size = thinker_config.vision_config.spatial_merge_size - tokens_per_second = getattr(thinker_config.vision_config, "tokens_per_second", 25) + elif modality == "video": + grid_t = data["grid_t"] + grid_h = data["grid_h"] + grid_w = data["grid_w"] + t_factor = data["t_factor"] - src_item = input_tokens - audio_seqlens = audio_feature_lengths - if not second_per_grid_ts: - second_per_grid_ts = [1] * video_grid_thw.shape[0] - audio_idx = 0 - video_idx = 0 - image_idx = 0 - new_src_item: list[int] = [] - llm_pos_ids_list: list[torch.Tensor] = [] + if not data["use_audio_in_video"]: + grid_indices = np.indices((grid_t, grid_h, grid_w)) + if t_factor != 1.0: + grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64) + llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx) + st = offset + grid_t * grid_h * grid_w + else: + pos_ids, token_count = self._compute_interleaved_positions(st_idx, data) + llm_pos_ids_list.append(pos_ids) + st = offset + token_count - idx = 0 - while idx < len(src_item): - new_src_item_len = len(new_src_item) - start_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - if src_item[idx] not in [audio_token_id, video_token_id, image_token_id]: - if use_audio_in_video and idx > 0: - if src_item[idx] == vision_end_token_id and src_item[idx - 1] == audio_end_token_id: - # processing the <|audio_eos|> before <|vision_eos|> - start_idx -= 1 - elif src_item[idx] == audio_start_token_id and src_item[idx - 1] == vision_start_token_id: - # processing the <|audio_bos|> after <|vision_eos|> - start_idx -= 1 - new_src_item.append(src_item[idx]) - llm_pos_ids = torch.tensor([start_idx], dtype=torch.long).expand(3, -1) - llm_pos_ids_list.append(llm_pos_ids) - elif src_item[idx] == audio_token_id: - assert audio_seqlens is not None - audio_seqlen = audio_seqlens[audio_idx] - place_num = ((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1 - new_src_item.extend([audio_token_id] * place_num) - llm_pos_ids = torch.arange(place_num).expand(3, -1) + start_idx - llm_pos_ids_list.append(llm_pos_ids) - audio_idx += 1 - elif src_item[idx] == image_token_id: - grid_t = image_grid_thw[image_idx][0] - grid_hs = image_grid_thw[:, 1] - grid_ws = image_grid_thw[:, 2] - t_index = (torch.arange(grid_t) * 1 * tokens_per_second).long() - llm_pos_ids = get_llm_pos_ids_for_vision( - start_idx, image_idx, spatial_merge_size, t_index, grid_hs, grid_ws - ) - llm_pos_ids_list.append(llm_pos_ids) - vision_seqlen = image_grid_thw[image_idx].prod() // (spatial_merge_size**2) - new_src_item.extend([image_token_id] * vision_seqlen) - image_idx += 1 - elif src_item[idx] == video_token_id and not use_audio_in_video: - grid_t = video_grid_thw[video_idx][0] - grid_hs = video_grid_thw[:, 1] - grid_ws = video_grid_thw[:, 2] - t_index = (torch.arange(grid_t) * second_per_grid_ts[video_idx] * tokens_per_second).long() - llm_pos_ids = get_llm_pos_ids_for_vision( - start_idx, video_idx, spatial_merge_size, t_index, grid_hs, grid_ws - ) - llm_pos_ids_list.append(llm_pos_ids) - vision_seqlen = video_grid_thw[video_idx].prod() // (spatial_merge_size**2) - new_src_item.extend([video_token_id] * vision_seqlen) - video_idx += 1 - else: - # read audio from video - assert audio_seqlens is not None - audio_seqlen = audio_seqlens[audio_idx] - vision_seqlen = video_grid_thw[video_idx].prod() // (spatial_merge_size**2) - grid_t = video_grid_thw[video_idx][0] - grid_h = video_grid_thw[video_idx][1] - grid_w = video_grid_thw[video_idx][2] - grid_hs = video_grid_thw[:, 1] - grid_ws = video_grid_thw[:, 2] - t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk) - t_index = (torch.arange(grid_t) * second_per_grid_ts[video_idx] * tokens_per_second).long() - t_index_split_chunk = split_list_into_ranges(t_index, t_ntoken_per_chunk) - place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1) + 2 - pure_audio_len = place_num - 2 - added_audio_len = 0 - audio_llm_pos_ids_list: list[torch.Tensor] = [] - for t_chunk in t_index_split_chunk: - vision_ntoken_per_chunk = len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2) - new_src_item.extend([video_token_id] * vision_ntoken_per_chunk) - vision_llm_pos_ids_list = get_llm_pos_ids_for_vision( - start_idx, - video_idx, - spatial_merge_size, - t_chunk, - grid_hs, - grid_ws, - ).split(1, dim=1) - llm_pos_ids_list.extend(vision_llm_pos_ids_list) - new_src_item.extend(min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) * [audio_token_id]) - audio_start_idx = ( - start_idx if len(audio_llm_pos_ids_list) == 0 else audio_llm_pos_ids_list[-1][0].item() + 1 - ) - if min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) > 0: - audio_llm_pos_ids_list = ( - torch.arange(min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)).expand(3, -1) - + audio_start_idx - ).split(1, dim=1) - else: - audio_llm_pos_ids_list = [] - added_audio_len += min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) - llm_pos_ids_list.extend(audio_llm_pos_ids_list) - if added_audio_len < pure_audio_len: - new_src_item.extend((pure_audio_len - added_audio_len) * [audio_token_id]) - audio_llm_pos_ids_list = ( - torch.arange(pure_audio_len - added_audio_len).expand(3, -1) + llm_pos_ids_list[-1].max() + 1 - ).split(1, dim=1) - llm_pos_ids_list.extend(audio_llm_pos_ids_list) - audio_idx += 1 - video_idx += 1 - # move to the next token - idx += len(new_src_item) - new_src_item_len + if st < len(input_tokens): + st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0 + text_len = len(input_tokens) - st + llm_pos_ids_list.append(np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx) - llm_positions = torch.cat(llm_pos_ids_list, dim=1) - mrope_position_delta = torch.cat(llm_pos_ids_list, dim=1).max() + 1 - len(src_item) + llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1) + mrope_position_delta = int(llm_positions.max()) + 1 - len(input_tokens) - return llm_positions, mrope_position_delta + return torch.from_numpy(llm_positions), mrope_position_delta def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) @@ -629,14 +649,12 @@ class Qwen2_5OmniThinkerForConditionalGeneration( # Check for audio-in-video: interleaved video and audio tokens # in the multimodal region. Only use the interleaved path when # needed; otherwise fall back to the default parent implementation. - # vLLM's _gather_mm_embeddings builds is_multimodal on CPU for merge - # indexing; bitwise ops with input_ids require the same device. - is_mm_device = is_multimodal.to(device=input_ids.device, non_blocking=True) video_token_id = self.config.video_token_index audio_token_id = self.config.audio_token_index - is_video = is_mm_device & (input_ids == video_token_id) - is_audio = is_mm_device & (input_ids == audio_token_id) + input_ids_cpu = input_ids.cpu() + is_video = is_multimodal & (input_ids_cpu == video_token_id) + is_audio = is_multimodal & (input_ids_cpu == audio_token_id) num_video = is_video.sum().item() num_audio = is_audio.sum().item() @@ -652,7 +670,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration( multimodal_embeddings, is_video, is_audio, - is_mm_device, + is_multimodal, num_video, num_audio, ) diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py index 17be417f3..44aaaf4d3 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py @@ -758,17 +758,11 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( use_audio_in_video = False if "video" in mm_kwargs: - non_none_items = [item for item in mm_kwargs["video"] if item is not None] - if non_none_items: - # Normal case: at least one non-cached item, read flag directly - use_audio_in_video = any(item["use_audio_in_video"].data for item in non_none_items) - elif "audio" in mm_prompt_updates: - # All video items are from cache (None); infer from prompt: - # use_audio_in_video=True means the prompt has no <|audio_pad|> - # placeholder (audio is embedded in video tokens instead) - tokenizer = self.info.get_tokenizer() - audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>") - use_audio_in_video = audio_pad_id not in prompt_ids + for item in mm_kwargs["video"]: + if item and item["use_audio_in_video"].data: + use_audio_in_video = True + else: + use_audio_in_video = False # for mutilmodality cache if any(item is None for item in mm_kwargs["video"]): video_token_id = self.info.get_hf_config().video_token_id