Add OmniMRotaryEmbedding for multimodal position computation

- Introduced OmniMRotaryEmbedding to extend MRotaryEmbedding with methods for computing input positions across various multimodal scenarios, including image, video, and audio inputs.
- Updated patch.py to replace the original MRotaryEmbedding with OmniMRotaryEmbedding in the module patching logic.
- Created new files for the OmniMRotaryEmbedding implementation and its associated methods.

Signed-off-by: [Your Name] <[Your Email]>

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-01-30 10:03:02 +00:00
1 parent 3208afd82b
commit 7aea8e8d91
17 files changed
+744 -57

No files matched your search

+27 -8
View File
@@ -39,17 +39,27 @@ class OmniAsyncMultiModalContentParser(AsyncMultiModalContentParser):
self._mm_processor_kwargs = mm_processor_kwargs
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
video = self._connector.fetch_video_async(video_url=video_url) if video_url else None
placeholder = self._tracker.add("video", video, uuid)
# OMNI: Follow upstream async pattern - create coroutine that resolves to (data, uuid)
coro = self._video_with_uuid_async(video_url, uuid)
placeholder = self._tracker.add("video", coro)
self._add_placeholder("video", placeholder)
# Extract audio from video if use_audio_in_video is True
if video_url and self._mm_processor_kwargs and self._mm_processor_kwargs.get("use_audio_in_video", False):
audio_coro = self._extract_audio_from_video_async(video_url)
audio_placeholder = self._tracker.add("audio", audio_coro, uuid)
audio_coro = self._audio_from_video_with_uuid_async(video_url, uuid)
audio_placeholder = self._tracker.add("audio", audio_coro)
self._add_placeholder("audio", audio_placeholder)
async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
"""Fetch video and return (video, uuid) tuple."""
video = await self._connector.fetch_video_async(video_url=video_url) if video_url else None
return video, uuid
async def _audio_from_video_with_uuid_async(self, video_url: str, uuid: str | None):
"""Extract audio from video and return (audio, uuid) tuple."""
audio = await self._extract_audio_from_video_async(video_url)
return audio, uuid
async def _extract_audio_from_video_async(self, video_url: str) -> tuple[np.ndarray, int | float]:
"""
Extract audio from video URL using librosa.
@@ -132,9 +142,17 @@ def parse_chat_messages_futures(
mm_processor_kwargs: dict[str, Any] | None = None,
) -> tuple[
list[ConversationMessage],
Awaitable[MultiModalDataDict | None],
MultiModalUUIDDict | None,
Awaitable[tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]],
]:
"""Parse chat messages and return conversation with multimodal data future.
OMNI: Updated to use upstream vLLM v0.15.0 API where resolve_items()
returns both mm_data and mm_uuids together as a tuple.
Returns:
Tuple of (conversation, mm_future) where mm_future resolves to
(mm_data, mm_uuids) when awaited.
"""
conversation: list[ConversationMessage] = []
mm_tracker = OmniAsyncMultiModalItemTracker(model_config)
@@ -155,7 +173,8 @@ def parse_chat_messages_futures(
_postprocess_messages(conversation)
return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
# OMNI: Use upstream resolve_items() which returns (mm_data, mm_uuids) tuple
return conversation, mm_tracker.resolve_items()
def _parse_chat_message_content(