Add OmniMRotaryEmbedding for multimodal position computation

- Introduced OmniMRotaryEmbedding to extend MRotaryEmbedding with methods for computing input positions across various multimodal scenarios, including image, video, and audio inputs.
- Updated patch.py to replace the original MRotaryEmbedding with OmniMRotaryEmbedding in the module patching logic.
- Created new files for the OmniMRotaryEmbedding implementation and its associated methods.

Signed-off-by: [Your Name] <[Your Email]>

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-01-30 10:03:02 +00:00
1 parent 3208afd82b
commit 7aea8e8d91
17 files changed
+744 -57

No files matched your search

@@ -189,6 +189,24 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
# OMNI: True if any attention backend handles KV cache update separately
# from forward() (i.e., forward_includes_kv_cache_update=False). When true,
# slot_mappings must use padded dimensions to match the key/value tensors.
from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec
has_separate_kv_update = not all(
all(g.backend.forward_includes_kv_cache_update for g in self.attn_groups[id])
for id, spec in enumerate(self.kv_cache_config.kv_cache_groups)
if not isinstance(spec.kv_cache_spec, EncoderOnlyAttentionSpec)
)
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
num_tokens_padded=num_tokens_padded if pad_attn or has_separate_kv_update else num_tokens_unpadded,
num_reqs_padded=(num_reqs_padded if pad_attn or has_separate_kv_update else num_reqs),
num_tokens_unpadded=num_tokens_unpadded,
ubatch_slices=ubatch_slices_padded,
)
attn_metadata, spec_decode_common_attn_metadata = self._build_attention_metadata(
num_tokens=num_tokens_unpadded,
num_tokens_padded=num_tokens_padded if pad_attn else None,
@@ -200,6 +218,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
use_spec_decode=use_spec_decode,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
slot_mappings=slot_mappings_by_group,
)
(
@@ -234,6 +253,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
cudagraph_runtime_mode=cudagraph_mode,
batch_descriptor=batch_desc,
ubatch_slices=ubatch_slices_padded,
slot_mapping=slot_mappings, # OMNI: required for KV cache operations
),
record_function_or_nullcontext("Forward"),
self.maybe_get_kv_connector_output(scheduler_output) as kv_connector_output,
@@ -539,6 +559,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
attn_metadata: PerLayerAttnMetadata | None = None
# OMNI: Get slot mappings before building attention metadata
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
num_tokens_padded=num_tokens,
num_reqs_padded=num_reqs_padded,
num_tokens_unpadded=num_tokens_unpadded,
ubatch_slices=ubatch_slices_padded,
)
# If force_attention is True, we always capture attention. Otherwise,
# it only happens for cudagraph_runtime_mode=FULL.
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
@@ -564,6 +592,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
max_query_len=max_query_len,
ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices,
for_cudagraph_capture=is_graph_capturing,
slot_mappings=slot_mappings_by_group,
)
with self.maybe_dummy_run_with_lora(
@@ -628,6 +657,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
cudagraph_runtime_mode=cudagraph_runtime_mode,
batch_descriptor=batch_desc,
ubatch_slices=ubatch_slices_padded,
slot_mapping=slot_mappings, # OMNI: required for KV cache operations
),
):
outputs = self.model(