Add OmniMRotaryEmbedding for multimodal position computation
- Introduced OmniMRotaryEmbedding to extend MRotaryEmbedding with methods for computing input positions across various multimodal scenarios, including image, video, and audio inputs. - Updated patch.py to replace the original MRotaryEmbedding with OmniMRotaryEmbedding in the module patching logic. - Created new files for the OmniMRotaryEmbedding implementation and its associated methods. Signed-off-by: [Your Name] <[Your Email]> Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
1 parent
3208afd82b
commit
7aea8e8d91
17 files changed
+744
-57
No files matched your search
@@ -552,6 +552,13 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
|
||||
attn_metadata: PerLayerAttnMetadata | None = None
|
||||
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
num_tokens_padded=num_tokens,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_unpadded=num_tokens_unpadded,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
)
|
||||
|
||||
# If force_attention is True, we always capture attention. Otherwise,
|
||||
# it only happens for cudagraph_runtime_mode=FULL.
|
||||
if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||||
@@ -577,6 +584,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
max_query_len=max_query_len,
|
||||
ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices,
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
@@ -641,6 +649,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
batch_descriptor=batch_desc,
|
||||
ubatch_slices=ubatch_slices_padded,
|
||||
slot_mapping=slot_mappings,
|
||||
),
|
||||
):
|
||||
if getattr(self.model, "talker", None) is not None and hasattr(self.model, "talker_mtp"):
|
||||
@@ -688,6 +697,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
num_tokens,
|
||||
use_cudagraphs=use_cudagraphs,
|
||||
is_graph_capturing=is_graph_capturing,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
|
||||
# We register layerwise NVTX hooks here after the first dynamo tracing is
|
||||
|
||||
Reference in new issue
Block a user