[Model] Add MiniMax H3 diffusion support (#5691)

This commit is contained in:
Isotr0py authored and GitHub committed 2026-08-03 14:10:40 +08:00
1 parent 727f90958e
commit 24741961eb
45 files changed
+7356 -31

No files matched your search

@@ -9,9 +9,11 @@ import torch
from .ring_globals import (
HAS_AITER,
HAS_FA3,
HAS_FA4,
HAS_FLASH_ATTN,
HAS_FLASHINFER,
fa3_fwd_func,
fa4_attn_func,
)
_scaled_dot_product_flash_attention = torch.ops.aten._scaled_dot_product_flash_attention
@@ -190,6 +192,40 @@ def fa3_forward(q, k, v, dropout_p, softmax_scale, causal, window_size, softcap,
flash_attn3_func_forward = fa3_forward
def flash_attn4_func_forward(
q,
k,
v,
dropout_p=0.0,
softmax_scale=None,
causal=False,
window_size=(-1, -1),
softcap=None,
alibi_slopes=None,
return_softmax=False,
):
"""FA4 forward pass returning the LSE required by Ring Attention."""
del dropout_p, alibi_slopes, return_softmax
assert HAS_FA4, "FA4 is not available"
assert fa4_attn_func is not None, "FA4 high-level API is not available"
left, right = window_size if window_size else (-1, -1)
out, softmax_lse = fa4_attn_func(
q,
k,
v,
softmax_scale=softmax_scale,
causal=causal,
window_size=(
None if left == -1 else left,
None if right == -1 else right,
),
softcap=softcap or 0.0,
return_lse=True,
)
return out, softmax_lse
def flash_attn_forward_aiter(
q,
k,