Add OmniMRotaryEmbedding for multimodal position computation

- Introduced OmniMRotaryEmbedding to extend MRotaryEmbedding with methods for computing input positions across various multimodal scenarios, including image, video, and audio inputs.
- Updated patch.py to replace the original MRotaryEmbedding with OmniMRotaryEmbedding in the module patching logic.
- Created new files for the OmniMRotaryEmbedding implementation and its associated methods.

Signed-off-by: [Your Name] <[Your Email]>

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-01-30 10:03:02 +00:00
1 parent 3208afd82b
commit 7aea8e8d91
17 files changed
+744 -57

No files matched your search

+6 -6
View File
@@ -1,10 +1,9 @@
import sys
from vllm.inputs.data import TokensPrompt as _OriginalTokensPrompt
# from vllm.model_executor.layers.rotary_embedding import (
# MRotaryEmbedding as _OriginalMRotaryEmbedding,
# )
from vllm.model_executor.layers.rotary_embedding import (
MRotaryEmbedding as _OriginalMRotaryEmbedding,
)
from vllm.v1.engine import EngineCoreOutput as _OriginalEngineCoreOutput
from vllm.v1.engine import EngineCoreOutputs as _OriginalEngineCoreOutputs
from vllm.v1.engine import EngineCoreRequest as _OriginalEngineCoreRequest
@@ -13,6 +12,7 @@ from vllm.v1.request import Request as _OriginalRequest
import vllm_omni.logger # noqa: F401
from vllm_omni.engine import OmniEngineCoreOutput, OmniEngineCoreOutputs, OmniEngineCoreRequest
from vllm_omni.inputs.data import OmniTokensPrompt
from vllm_omni.model_executor.layers.rotary_embedding import OmniMRotaryEmbedding
from vllm_omni.request import OmniRequest
for module_name, module in sys.modules.items():
@@ -25,8 +25,8 @@ for module_name, module in sys.modules.items():
module.EngineCoreOutputs = OmniEngineCoreOutputs
if hasattr(module, "TokensPrompt") and module.TokensPrompt == _OriginalTokensPrompt:
module.TokensPrompt = OmniTokensPrompt
# if hasattr(module, "MRotaryEmbedding") and module.MRotaryEmbedding == _OriginalMRotaryEmbedding:
# module.MRotaryEmbedding = MRotaryEmbedding
if hasattr(module, "MRotaryEmbedding") and module.MRotaryEmbedding == _OriginalMRotaryEmbedding:
module.MRotaryEmbedding = OmniMRotaryEmbedding
if hasattr(module, "Request") and module.Request == _OriginalRequest:
module.Request = OmniRequest
if hasattr(module, "EngineCoreRequest") and module.EngineCoreRequest == _OriginalEngineCoreRequest: