[Model] Add MiniMax H3 diffusion support (#5691)
This commit is contained in:
1 parent
727f90958e
commit
24741961eb
45 files changed
+7356
-31
No files matched your search
@@ -9,9 +9,11 @@ import torch
|
||||
from .ring_globals import (
|
||||
HAS_AITER,
|
||||
HAS_FA3,
|
||||
HAS_FA4,
|
||||
HAS_FLASH_ATTN,
|
||||
HAS_FLASHINFER,
|
||||
fa3_fwd_func,
|
||||
fa4_attn_func,
|
||||
)
|
||||
|
||||
_scaled_dot_product_flash_attention = torch.ops.aten._scaled_dot_product_flash_attention
|
||||
@@ -190,6 +192,40 @@ def fa3_forward(q, k, v, dropout_p, softmax_scale, causal, window_size, softcap,
|
||||
flash_attn3_func_forward = fa3_forward
|
||||
|
||||
|
||||
def flash_attn4_func_forward(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
dropout_p=0.0,
|
||||
softmax_scale=None,
|
||||
causal=False,
|
||||
window_size=(-1, -1),
|
||||
softcap=None,
|
||||
alibi_slopes=None,
|
||||
return_softmax=False,
|
||||
):
|
||||
"""FA4 forward pass returning the LSE required by Ring Attention."""
|
||||
del dropout_p, alibi_slopes, return_softmax
|
||||
assert HAS_FA4, "FA4 is not available"
|
||||
assert fa4_attn_func is not None, "FA4 high-level API is not available"
|
||||
|
||||
left, right = window_size if window_size else (-1, -1)
|
||||
out, softmax_lse = fa4_attn_func(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
window_size=(
|
||||
None if left == -1 else left,
|
||||
None if right == -1 else right,
|
||||
),
|
||||
softcap=softcap or 0.0,
|
||||
return_lse=True,
|
||||
)
|
||||
return out, softmax_lse
|
||||
|
||||
|
||||
def flash_attn_forward_aiter(
|
||||
q,
|
||||
k,
|
||||
|
||||
Reference in new issue
Block a user