Add OmniMRotaryEmbedding for multimodal position computation
- Introduced OmniMRotaryEmbedding to extend MRotaryEmbedding with methods for computing input positions across various multimodal scenarios, including image, video, and audio inputs. - Updated patch.py to replace the original MRotaryEmbedding with OmniMRotaryEmbedding in the module patching logic. - Created new files for the OmniMRotaryEmbedding implementation and its associated methods. Signed-off-by: [Your Name] <[Your Email]> Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
3208afd82b
commit
7aea8e8d91
17 files changed
+744
-57
No files matched your search
@@ -189,6 +189,24 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
|
||||
|
||||
# OMNI: True if any attention backend handles KV cache update separately
|
||||
# from forward() (i.e., forward_includes_kv_cache_update=False). When true,
|
||||
# slot_mappings must use padded dimensions to match the key/value tensors.
|
||||
from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec
|
||||
|
||||
has_separate_kv_update = not all(
|
||||
all(g.backend.forward_includes_kv_cache_update for g in self.attn_groups[id])
|
||||
for id, spec in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
if not isinstance(spec.kv_cache_spec, EncoderOnlyAttentionSpec)
|
||||
)
|
||||
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
num_tokens_padded=num_tokens_padded if pad_attn or has_separate_kv_update else num_tokens_unpadded,
|
||||
num_reqs_padded=(num_reqs_padded if pad_attn or has_separate_kv_update else num_reqs),
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
)
|
||||
|
||||
attn_metadata, spec_decode_common_attn_metadata = self._build_attention_metadata(
|
||||
num_tokens=num_tokens_unpadded,
|
||||
num_tokens_padded=num_tokens_padded if pad_attn else None,
|
||||
@@ -200,6 +218,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
use_spec_decode=use_spec_decode,
|
||||
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
|
||||
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
|
||||
(
|
||||
@@ -234,6 +253,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
cudagraph_runtime_mode=cudagraph_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
slot_mapping=slot_mappings, # OMNI: required for KV cache operations
|
||||
),
|
||||
record_function_or_nullcontext("Forward"),
|
||||
self.maybe_get_kv_connector_output(scheduler_output) as kv_connector_output,
|
||||
@@ -539,6 +559,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
|
||||
attn_metadata: PerLayerAttnMetadata | None = None
|
||||
|
||||
# OMNI: Get slot mappings before building attention metadata
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
num_tokens_padded=num_tokens,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
)
|
||||
|
||||
# If force_attention is True, we always capture attention. Otherwise,
|
||||
# it only happens for cudagraph_runtime_mode=FULL.
|
||||
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||
@@ -564,6 +592,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
max_query_len=max_query_len,
|
||||
ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices,
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
@@ -628,6 +657,7 @@ class GPUGenerationModelRunner(OmniGPUModelRunner):
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
slot_mapping=slot_mappings, # OMNI: required for KV cache operations
|
||||
),
|
||||
):
|
||||
outputs = self.model(
|
||||
|
||||
Reference in new issue
Block a user