[Perf][Diffusion] MiniMax-H3: opt into packed varlen attention on NPU to eliminate quadratic mask materialization (#5891)

Signed-off-by: brandneway <gyuan4892@gmail.com>
This commit is contained in:
ooooooye authored and GitHub committed 2026-08-10 19:47:03 +08:00
1 parent 5e4b9f8c6e
commit 740f45abcd
7 files changed
+638 -8

No files matched your search

@@ -20,6 +20,17 @@ class AttentionBackend(ABC):
# avoid a slower masked-attention plan when tail padding is not semantic.
supports_prefix_kv_slicing: bool = False
@classmethod
def supports_packed_mask_free(cls) -> bool:
"""Whether packed attention never reads attn_mask on this platform.
When True, models that pack a [real, pad] two-document layout and
carry cu_seqlens/max_seqlen in ``AttentionMetadata.extra`` may skip
constructing the padding mask entirely. Backends whose mask-free
behavior is platform-dependent must check current_omni_platform.
"""
return False
# ``OmniPlatformEnum`` values this backend runs on; None means unrestricted.
# Platform resolution rejects an explicit selection outside this set, so a
# hardware-specific backend fails before the model is built.
@@ -121,6 +132,17 @@ class AttentionMetadata:
# the packed cu_seqlens tensors.
# "valid_kv_length": int — contiguous valid K/V prefix length for a
# backend that advertises supports_prefix_kv_slicing.
# "npu_attn_varlen": bool — model opt-in for the NPU packed varlen path
# (TND npu_fusion_attention driven by cu_seqlens, mask never read).
# Requires the [real, pad] two-document packing contract; see
# FlashAttentionImpl._forward_varlen_packed_npu.
# "laser_input_scale": float — model opt-in input pre-scale for the NPU
# ascend_laser_attention path. The kernel stores unscaled QK^T in an
# fp16 workspace, so outlier activations overflow 65504 into NaN rows;
# with this set (>1), q/k/v are divided by the factor before the op,
# the kernel scale_value is multiplied by its square, and the output
# is scaled back (exact for power-of-two factors). Absent means no
# pre-scaling. See FlashAttentionImpl._forward_prefix_kv_slice_npu.
# Piecewise attention metadata (mixed causal/full masks).
# full_attn_spans: per-sample [start, end) spans in global coordinates using full attention.
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from functools import partial
import torch
@@ -16,6 +17,7 @@ from vllm_omni.diffusion.attention.backends.utils.piecewise_attn import (
piecewise_attn,
)
from vllm_omni.diffusion.config import get_current_diffusion_config_or_none
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
@@ -24,6 +26,15 @@ class FlashAttentionBackend(AttentionBackend):
accept_output_buffer: bool = True
supports_piecewise_spans: bool = True
@classmethod
def supports_packed_mask_free(cls) -> bool:
# CUDA: forward_cuda dispatches the packed cu_seqlens varlen path
# before ever reading attn_mask. NPU: forward_fa_npu honors the
# npu_attn_varlen opt-in and rebuilds the padding mask itself if the
# packed contract fails. XPU reads attn_mask, so models must keep
# constructing it there.
return current_omni_platform.is_cuda() or current_omni_platform.is_npu()
@classmethod
def supports_attention_mask(cls) -> bool:
return True
@@ -415,7 +426,36 @@ class FlashAttentionImpl(AttentionImpl):
"For installation details, see https://gitcode.com/Ascend/MindIE-SD"
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
)
# Opt-in mask-free paths (mirror the CUDA cu_seqlens behavior): the
# model marks extra["npu_attn_varlen"] and carries packed metadata, so
# the padding document is excluded without reading or materializing
# the quadratic full_qk mask.
# - default: packed varlen via mindiesd attention_forward_varlen;
# - when MINDIE_SD_FA_TYPE=ascend_laser_attention: prefix K/V
# slicing via mindiesd attention_forward (so the env-selected op
# type is honored; the TND varlen op cannot be).
extra = attn_metadata.extra if attn_metadata else {}
if extra.get("npu_attn_varlen", False):
if os.environ.get("MINDIE_SD_FA_TYPE") == "ascend_laser_attention":
out = self._forward_prefix_kv_slice_npu(query, key, value, extra)
else:
out = self._forward_varlen_packed_npu(query, key, value, extra)
if out is not None:
return out
attention_mask = attn_metadata.attn_mask if attn_metadata else None
if attention_mask is None and extra.get("npu_attn_varlen", False):
# Models skip mask construction when this opt-in is set
# (FlashAttentionBackend.supports_packed_mask_free). The packed
# paths above declined (contract mismatch), so rebuild the padding
# mask here to keep the masked fallback correct.
used = extra.get("valid_kv_length")
if not isinstance(used, int) or not 0 < used <= query.shape[1]:
raise ValueError(
"npu_attn_varlen packed metadata is unusable and no attn_mask "
f"was constructed (valid_kv_length={used!r}, seq_len={query.shape[1]}); "
"refusing to run unmasked attention over padding rows."
)
attention_mask = torch.arange(query.shape[1], device=query.device)[None] < used
# NPU aclnnFlashAttentionScore requires mask shape to be one of:
# [B, N, Sq, Skv], [B, 1, Sq, Skv], [1, 1, Sq, Skv], or [Sq, Skv]
@@ -433,3 +473,161 @@ class FlashAttentionImpl(AttentionImpl):
op_type="fused_attn_score",
layout=layout,
)
def _resolve_packed_seq_npu(
self,
query: torch.Tensor,
key: torch.Tensor,
extra: dict,
) -> tuple[list[int], list[int]] | None:
"""Resolve packed document boundaries (cumulative end offsets per doc).
Returns (seq_q, seq_k), e.g. ([used_q, total_q], [used_k, total_k]),
or None unless the metadata matches the exact contract MiniMax-H3
produces:
- q/k/v are [1, T, N, D] (BSND, single packed batch);
- cu_seqlens describe a [real, pad] two-document packing, with the
padding document as a strict suffix;
- max_seqlen_q/k are Python ints equal to the real document length
(so boundaries are derived without any device sync).
"""
if self.causal:
return None
packed_keys = ("cu_seqlens_q", "cu_seqlens_k", "max_seqlen_q", "max_seqlen_k")
if any(k not in extra for k in packed_keys):
return None
cu_q, cu_k = extra["cu_seqlens_q"], extra["cu_seqlens_k"]
used_q, used_k = extra["max_seqlen_q"], extra["max_seqlen_k"]
if query.shape[0] != 1 or key.shape[0] != 1:
return None
if not isinstance(used_q, int) or not isinstance(used_k, int):
return None
total_q, total_k = query.shape[1], key.shape[1]
# .shape is host-side metadata: counting documents never syncs.
if cu_q.shape[0] != cu_k.shape[0] or cu_q.shape[0] > 3:
return None
if not (0 < used_q <= total_q) or not (0 < used_k <= total_k):
return None
if used_q == total_q and used_k == total_k:
# No padding document: one full-length document.
return [total_q], [total_k]
if cu_q.shape[0] == 3 and used_q >= total_q - used_q and used_k >= total_k - used_k:
# [real, pad] packing: the real document must be the longer one
# (consistent with the max_seqlen naming).
return [used_q, total_q], [used_k, total_k]
return None
def _forward_varlen_packed_npu(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
extra: dict,
) -> torch.Tensor | None:
"""Packed varlen attention on NPU via mindiesd attention_forward_varlen.
Returns None (caller falls back to the mask path) when the packed
contract does not hold; see _resolve_packed_seq_npu.
"""
resolved = self._resolve_packed_seq_npu(query, key, extra)
if resolved is None:
return None
seq_q, seq_k = resolved
try:
from mindiesd import attention_forward_varlen
except ImportError:
raise ImportError(
"FlashAttentionBackend NPU implementation requires MindIE-SD. "
"Please install MindIE-SD to enable NPU attention support. "
"For installation details, see https://gitcode.com/Ascend/MindIE-SD"
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
)
q = query.squeeze(0) # [T, N, D] == TND
k = key.squeeze(0)
v = value.squeeze(0)
# attention_forward_varlen wants cu_seqlens as host lists of cumulative
# offsets and forwards cu[1:] as actual_seq_qlen/actual_seq_kvlen.
out = attention_forward_varlen(
q,
k,
v,
[0, *seq_q],
[0, *seq_k],
softmax_scale=self.softmax_scale,
)
return out.unsqueeze(0)
def _forward_prefix_kv_slice_npu(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
extra: dict,
) -> torch.Tensor | None:
"""Mask-free attention by slicing K/V to the valid prefix (zero-copy
views), then mindiesd attention_forward without a mask.
Same numerical contract as the varlen path: the padding document is a
strict suffix, so dropping it from K/V is identical to masking it out.
Query keeps full length; outputs on padding rows are never consumed
downstream. Returns None when the packed contract does not hold.
Used when MINDIE_SD_FA_TYPE=ascend_laser_attention: the TND varlen op
cannot honor the env-selected op type, while attention_forward can.
The laser kernel stores unscaled S=QK^T in an fp16 GM workspace, so
bf16 activations with large outliers overflow 65504 into ±inf and the
affected rows turn NaN. Models may opt into exact power-of-two input
pre-scaling via extra["laser_input_scale"] (see abstract.py): q/k/v
are divided by the factor, the kernel scale_value is multiplied by its
square, and the output is scaled back. Absent or invalid factor means
no pre-scaling.
"""
resolved = self._resolve_packed_seq_npu(query, key, extra)
if resolved is None:
return None
_, seq_k = resolved
used_k = seq_k[0] # real document length (first cumulative end)
try:
from mindiesd import attention_forward
except ImportError:
raise ImportError(
"FlashAttentionBackend NPU implementation requires MindIE-SD. "
"Please install MindIE-SD to enable NPU attention support. "
"For installation details, see https://gitcode.com/Ascend/MindIE-SD"
"Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA"
)
# mindiesd always takes BSND input regardless of `layout`; the arg
# selects the op-internal layout and laser only supports BNSD, so do
# NOT forward the model's qkv_layout ("BSND" for MiniMax-H3) here.
layout = "BNSD"
key = key[:, :used_k]
value = value[:, :used_k]
input_scale = extra.get("laser_input_scale")
preserve_input_range = isinstance(input_scale, (int, float)) and input_scale > 1
if preserve_input_range:
query = query / input_scale
key = key / input_scale
value = value / input_scale
scale = self.softmax_scale
if preserve_input_range:
scale *= input_scale**2
def _postprocess(out: torch.Tensor) -> torch.Tensor:
return out * input_scale if preserve_input_range else out
return _postprocess(
attention_forward(
query,
key,
value,
attn_mask=None,
opt_mode="manual",
op_type="fused_attn_score", # MINDIE_SD_FA_TYPE env overrides this
layout=layout,
scale=scale,
)
)
@@ -107,6 +107,15 @@ MINIMAX_H3_FP32_BUFFER_NAMES = frozenset({"rope.inv_freq"})
# lookup and masked out afterwards).
MINIMAX_H3_ADALN_MODALITY_NUM = 3
# Opt-in fp16-range protection for the NPU ascend_laser_attention kernel
# (consumed only via the "laser_input_scale" extra key; other backends and
# platforms ignore it). The kernel stores unscaled QK^T in an fp16 GM
# workspace, and H3's outlier activations (per-element amax in the hundreds)
# push dot products past fp16 max 65504, turning whole 128-row blocks NaN.
# 256 is a power of two, so pre-dividing q/k/v and the compensating
# kernel-scale/output multiplies are exact in floating point.
MINIMAX_H3_LASER_INPUT_SCALE = 256.0
def _required_kwarg(kwargs: dict[str, Any], key: str) -> Any:
if key not in kwargs or kwargs[key] is None:
@@ -402,11 +411,16 @@ class MiniMaxH3Attention(nn.Module):
used = min(max_seqlen, packed_total)
attn_mask = None
# Ring attention can dispatch to a different implementation from the
# configured backend, so this no-mask fast path is local-only.
prefix_slice = (
not getattr(self.attention, "use_ring", False) and self.attention.attn_backend.supports_prefix_kv_slicing
# configured backend, so the no-mask fast paths are local-only.
# supports_prefix_kv_slicing: backend slices K/V itself (cuDNN).
# supports_packed_mask_free: backend consumes the packed metadata
# without ever reading attn_mask (CUDA packed varlen, NPU
# npu_attn_varlen opt-in with its own fallback rebuild).
no_mask = not getattr(self.attention, "use_ring", False) and (
self.attention.attn_backend.supports_prefix_kv_slicing
or self.attention.attn_backend.supports_packed_mask_free()
)
if used < packed_total and not prefix_slice:
if used < packed_total and not no_mask:
attn_mask = torch.arange(packed_total, device=q.device)[None] < used
metadata = AttentionMetadata(
attn_mask=attn_mask,
@@ -416,6 +430,15 @@ class MiniMaxH3Attention(nn.Module):
"max_seqlen_q": max_seqlen,
"max_seqlen_k": max_seqlen,
"valid_kv_length": used,
# Opt the NPU flash backend into the packed varlen path so the
# quadratic full_qk mask is never materialized. Ring attention
# is excluded: it keeps the aligned padding rows for its
# fixed-size P2P buffers and still needs the mask.
"npu_attn_varlen": not getattr(self.attention, "use_ring", False),
# fp16-range protection for the ascend_laser_attention kernel
# (see MINIMAX_H3_LASER_INPUT_SCALE). Ignored by every other
# backend/path.
"laser_input_scale": MINIMAX_H3_LASER_INPUT_SCALE,
},
video_layout=video_layout,
)