feat(minimax-h3): enable 2x RTX 4090/5090 with DLO (#5764)
Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Signed-off-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
This commit is contained in:
1 parent
b4581b293b
commit
1c2a81f6d8
28 files changed
+1658
-162
No files matched your search
@@ -15,6 +15,10 @@ class AttentionBackend(ABC):
|
||||
|
||||
accept_output_buffer: bool = False
|
||||
supports_piecewise_spans: bool = False
|
||||
# The backend can represent a contiguous valid K/V prefix by slicing the
|
||||
# tensors instead of materializing a padding mask. Models may use this to
|
||||
# avoid a slower masked-attention plan when tail padding is not semantic.
|
||||
supports_prefix_kv_slicing: bool = False
|
||||
|
||||
@classmethod
|
||||
def supports_attention_mask(cls) -> bool:
|
||||
@@ -83,6 +87,8 @@ class AttentionMetadata:
|
||||
# variable-length query/key sequences for FlashAttention.
|
||||
# "max_seqlen_q" / "max_seqlen_k": maximum sequence lengths paired with
|
||||
# the packed cu_seqlens tensors.
|
||||
# "valid_kv_length": int — contiguous valid K/V prefix length for a
|
||||
# backend that advertises supports_prefix_kv_slicing.
|
||||
|
||||
# Piecewise attention metadata (mixed causal/full masks).
|
||||
# full_attn_spans: per-sample [start, end) spans in global coordinates using full attention.
|
||||
|
||||
@@ -17,6 +17,7 @@ logger = init_logger(__name__)
|
||||
|
||||
class CuDNNAttentionBackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
supports_prefix_kv_slicing: bool = True
|
||||
|
||||
@classmethod
|
||||
def supports_attention_mask(cls) -> bool:
|
||||
@@ -61,7 +62,26 @@ class CuDNNAttentionImpl(AttentionImpl):
|
||||
) -> torch.Tensor:
|
||||
attention_mask = None
|
||||
if attn_metadata:
|
||||
attention_mask = _maybe_reshape_attn_mask(query, key, attn_metadata.attn_mask, mask_mode="broadcast_k")
|
||||
valid_kv_length = attn_metadata.extra.get("valid_kv_length")
|
||||
if attn_metadata.attn_mask is None and isinstance(valid_kv_length, int):
|
||||
if not 0 < valid_kv_length <= key.shape[1]:
|
||||
raise ValueError(
|
||||
"valid_kv_length must be within the K/V sequence, "
|
||||
f"got {valid_kv_length} for length {key.shape[1]}"
|
||||
)
|
||||
# A contiguous valid prefix is mathematically equivalent to a
|
||||
# broadcast key-padding mask. Slicing K/V keeps Q/output in the
|
||||
# checkpoint's aligned layout while letting cuDNN select its
|
||||
# much faster mask-free FMHA plan.
|
||||
key = key[:, :valid_kv_length]
|
||||
value = value[:, :valid_kv_length]
|
||||
else:
|
||||
attention_mask = _maybe_reshape_attn_mask(
|
||||
query,
|
||||
key,
|
||||
attn_metadata.attn_mask,
|
||||
mask_mode="broadcast_k",
|
||||
)
|
||||
|
||||
enable_gqa = query.shape[2] != key.shape[2]
|
||||
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
|
||||
|
||||
Reference in new issue
Block a user