[Perf]Boogu-Image Skip dense attention masks (#6871)

Signed-off-by: zhuhh97 <zhuhonghan@huawei.com>
This commit is contained in:
Honghan Zhu authored and GitHub committed 2026-09-08 00:53:20 -07:00
1 parent c704aeccae
commit c086554d1f
3 files changed
+62 -9

No files matched your search

+29 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -31,6 +31,34 @@ if TYPE_CHECKING:
Style = Literal["colwise", "colwise_rep", "rowwise", "rowwise_rep", "replicate"]
def make_attention_mask(
hidden_states: torch.Tensor,
seq_lengths: list[int],
) -> torch.Tensor | None:
"""Build a padding mask only when the batch contains padding.
``seq_lengths`` must be non-empty and contain valid sequence lengths.
Returns a boolean mask of shape ``(len(seq_lengths), max(seq_lengths))``
on the device of ``hidden_states``, with True marking valid tokens,
or None when all sequence lengths are equal.
Passing an all-true CUDA mask makes attention backends inspect a device
scalar to select the dense path, which introduces a host-device sync.
"""
max_seq_len = max(seq_lengths)
if all(seq_len == max_seq_len for seq_len in seq_lengths):
return None
attention_mask = hidden_states.new_zeros(
len(seq_lengths),
max_seq_len,
dtype=torch.bool,
)
for i, seq_len in enumerate(seq_lengths):
attention_mask[i, :seq_len] = True
return attention_mask
def replace_linear_class(
linear: nn.Linear,
style: Style = "replicate",