[Blackwell] Add CUDNN_ATTN and FLASHINFER_ATTN backends for diffusion (auto-route) (#3079)

Signed-off-by: lishunyang <lishunyang12@163.com>
Signed-off-by: Zhang <jianmusings@gmail.com>
Signed-off-by: Zhang Jian <jianmusings@gmail.com>
Co-authored-by: frauttauteffasu <247219357+frauttauteffasu@users.noreply.github.com>
Co-authored-by: Zhang Jian <jianmusings@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-05-28 11:09:25 +08:00
1 parent 1903cee56a
commit 373d3c7476
11 files changed
+859 -41

No files matched your search

@@ -67,13 +67,20 @@ def _make_rms_norm(hidden_size: int, *, eps: float, elementwise_affine: bool) ->
def apply_interleaved_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
cos, sin = freqs
x_real, x_imag = x.unflatten(2, (-1, 2)).unbind(-1) # [B, S, C // 2]
# Concrete pair count instead of -1 keeps SDPA shape static under torch.compile.
x_real, x_imag = x.unflatten(2, (x.shape[2] // 2, 2)).unbind(-1) # [B, S, C // 2]
x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2)
out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype)
return out
def apply_split_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
def apply_split_rotary_emb(
x: torch.Tensor,
freqs: tuple[torch.Tensor, torch.Tensor],
*,
head_dim: int,
) -> torch.Tensor:
# `head_dim` is plumbed in (not inferred via `-1`) so SDPA shape stays static under torch.compile.
cos, sin = freqs
x_dtype = x.dtype
@@ -83,7 +90,7 @@ def apply_split_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Ten
# The cos/sin batch dim may only be broadcastable, so take batch size from x
b = x.shape[0]
_, h, t, _ = cos.shape
x = x.reshape(b, t, h, -1).swapaxes(1, 2)
x = x.reshape(b, t, h, head_dim).swapaxes(1, 2)
needs_reshape = True
# Split last dim (2*r) into (d=2, r)
@@ -110,7 +117,7 @@ def apply_split_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Ten
out = out.reshape(*out.shape[:-2], last)
if needs_reshape:
out = out.swapaxes(1, 2).reshape(b, t, -1)
out = out.swapaxes(1, 2).reshape(b, t, h * head_dim)
out = out.to(dtype=x_dtype)
return out
@@ -488,12 +495,14 @@ class LTX2AudioVideoAttnProcessor:
key, key_rotary_emb if key_rotary_emb is not None else query_rotary_emb
)
elif attn.rope_type == "split":
query = apply_split_rotary_emb(query, query_rotary_emb)
key = apply_split_rotary_emb(key, key_rotary_emb if key_rotary_emb is not None else query_rotary_emb)
query = apply_split_rotary_emb(query, query_rotary_emb, head_dim=attn.head_dim)
key = apply_split_rotary_emb(
key, key_rotary_emb if key_rotary_emb is not None else query_rotary_emb, head_dim=attn.head_dim
)
query = query.unflatten(2, (attn.heads, -1))
key = key.unflatten(2, (attn.heads, -1))
value = value.unflatten(2, (attn.heads, -1))
query = query.unflatten(2, (attn.heads, attn.head_dim))
key = key.unflatten(2, (attn.heads, attn.head_dim))
value = value.unflatten(2, (attn.heads, attn.head_dim))
attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None
hidden_states = attn.attn(query, key, value, attn_metadata)
@@ -502,7 +511,7 @@ class LTX2AudioVideoAttnProcessor:
# LTX-2.3: per-head gated attention
if attn.to_gate_logits is not None:
hidden_states = hidden_states.unflatten(2, (attn.heads, -1)) # [B, T, H, D]
hidden_states = hidden_states.unflatten(2, (attn.heads, attn.head_dim)) # [B, T, H, D]
# 2.0 * sigmoid so zero-init gates produce 1.0 (identity)
gates = 2.0 * torch.sigmoid(gate_logits) # [B, T, H]
hidden_states = hidden_states * gates.unsqueeze(-1)
@@ -1391,9 +1400,11 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module):
# Reshape freqs to be compatible with multi-head attention
b = cos_freq.shape[0]
t = cos_freq.shape[1]
# Concrete per-head dim instead of -1 — see apply_split_rotary_emb
r = self.dim // self.num_attention_heads // 2
cos_freq = cos_freq.reshape(b, t, self.num_attention_heads, -1)
sin_freq = sin_freq.reshape(b, t, self.num_attention_heads, -1)
cos_freq = cos_freq.reshape(b, t, self.num_attention_heads, r)
sin_freq = sin_freq.reshape(b, t, self.num_attention_heads, r)
cos_freqs = torch.swapaxes(cos_freq, 1, 2) # (B,H,T,D//2)
sin_freqs = torch.swapaxes(sin_freq, 1, 2) # (B,H,T,D//2)
@@ -157,6 +157,9 @@ class _VideoAudioScheduler:
class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
# Audio is diffused jointly with video; warmup must size audio tokens.
support_audio_output = True
def __init__(
self,
*,
@@ -115,6 +115,9 @@ class LTX23Pipeline(nn.Module, ProgressBarMixin):
- CPU offloading: text encoder, connectors, VAE, vocoder stay on CPU
"""
# Audio is diffused jointly with video; warmup must size audio tokens.
support_audio_output = True
def __init__(
self,
*,