[Model] MiniCPM-o 4.5: reuse FlashAttention unpadding metadata (#5165)

Signed-off-by: 道路自信 <182356120+daoluzixin@users.noreply.github.com>
Co-authored-by: 道路自信 <182356120+daoluzixin@users.noreply.github.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-08-07 10:25:04 +08:00
1 parent a874b8e09b
commit 6ec9033065
2 files changed
+412 -11

No files matched your search

@@ -770,8 +770,11 @@ AutoImageProcessor.register(MiniCPMOConfig, MiniCPMVImageProcessor, exist_ok=Tru
# ============== SigLIP Vision Transformer Classes ==============
FlashAttentionUnpaddingMetadata: TypeAlias = tuple[torch.Tensor, torch.Tensor, int]
# Copied from transformers.models.llama.modeling_llama._get_unpad_data
def _get_unpad_data(attention_mask):
def _get_unpad_data(attention_mask: torch.Tensor) -> FlashAttentionUnpaddingMetadata:
seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
max_seqlen_in_batch = seqlens_in_batch.max().item()
@@ -1116,6 +1119,7 @@ class SiglipFlashAttention2(SiglipAttention):
past_key_value: tuple[torch.Tensor] | None = None,
output_attentions: bool = False,
use_cache: bool = False,
unpadding_metadata: FlashAttentionUnpaddingMetadata | None = None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
output_attentions = False
@@ -1178,7 +1182,13 @@ class SiglipFlashAttention2(SiglipAttention):
value_states = value_states.to(target_dtype)
attn_output = self._flash_attention_forward(
query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
query_states,
key_states,
value_states,
attention_mask,
q_len,
dropout=dropout_rate,
unpadding_metadata=unpadding_metadata,
)
attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
@@ -1190,7 +1200,15 @@ class SiglipFlashAttention2(SiglipAttention):
return attn_output, attn_weights
def _flash_attention_forward(
self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
self,
query_states,
key_states,
value_states,
attention_mask,
query_length,
dropout=0.0,
softmax_scale=None,
unpadding_metadata: FlashAttentionUnpaddingMetadata | None = None,
):
"""
Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
@@ -1218,7 +1236,12 @@ class SiglipFlashAttention2(SiglipAttention):
if attention_mask is not None:
batch_size = query_states.shape[0]
query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
query_states, key_states, value_states, attention_mask, query_length
query_states,
key_states,
value_states,
attention_mask,
query_length,
unpadding_metadata=unpadding_metadata,
)
cu_seqlens_q, cu_seqlens_k = cu_seq_lens
@@ -1245,8 +1268,18 @@ class SiglipFlashAttention2(SiglipAttention):
return attn_output
def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
def _upad_input(
self,
query_layer,
key_layer,
value_layer,
attention_mask,
query_length,
unpadding_metadata: FlashAttentionUnpaddingMetadata | None = None,
):
if unpadding_metadata is None:
unpadding_metadata = _get_unpad_data(attention_mask)
indices_k, cu_seqlens_k, max_seqlen_in_batch_k = unpadding_metadata
batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
key_layer = index_first_axis(
@@ -1316,6 +1349,7 @@ class SiglipEncoderLayer(nn.Module):
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
output_attentions: bool | None = False,
unpadding_metadata: FlashAttentionUnpaddingMetadata | None = None,
) -> tuple[torch.FloatTensor]:
"""
Args:
@@ -1330,11 +1364,19 @@ class SiglipEncoderLayer(nn.Module):
residual = hidden_states
hidden_states = self.layer_norm1(hidden_states)
hidden_states, attn_weights = self.self_attn(
hidden_states=hidden_states,
attention_mask=attention_mask,
output_attentions=output_attentions,
)
if self._use_flash_attention_2:
hidden_states, attn_weights = self.self_attn(
hidden_states=hidden_states,
attention_mask=attention_mask,
output_attentions=output_attentions,
unpadding_metadata=unpadding_metadata,
)
else:
hidden_states, attn_weights = self.self_attn(
hidden_states=hidden_states,
attention_mask=attention_mask,
output_attentions=output_attentions,
)
hidden_states = residual + hidden_states
residual = hidden_states
@@ -1435,6 +1477,7 @@ class SiglipEncoder(nn.Module):
self.config = config
self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
self.gradient_checkpointing = False
self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
# Ignore copy
def forward(
@@ -1475,6 +1518,9 @@ class SiglipEncoder(nn.Module):
all_attentions = () if output_attentions else None
hidden_states = inputs_embeds
unpadding_metadata = (
_get_unpad_data(attention_mask) if self._use_flash_attention_2 and attention_mask is not None else None
)
for encoder_layer in self.layers:
if output_hidden_states:
encoder_states = encoder_states + (hidden_states,)
@@ -1484,12 +1530,14 @@ class SiglipEncoder(nn.Module):
hidden_states,
attention_mask,
output_attentions,
unpadding_metadata,
)
else:
layer_outputs = encoder_layer(
hidden_states,
attention_mask,
output_attentions=output_attentions,
unpadding_metadata=unpadding_metadata,
)
hidden_states = layer_outputs[0]