[Blackwell] Add CUDNN_ATTN and FLASHINFER_ATTN backends for diffusion (auto-route) (#3079)
Signed-off-by: lishunyang <lishunyang12@163.com> Signed-off-by: Zhang <jianmusings@gmail.com> Signed-off-by: Zhang Jian <jianmusings@gmail.com> Co-authored-by: frauttauteffasu <247219357+frauttauteffasu@users.noreply.github.com> Co-authored-by: Zhang Jian <jianmusings@gmail.com>
This commit is contained in:
11 files changed
+859
-41
No files matched your search
@@ -67,13 +67,20 @@ def _make_rms_norm(hidden_size: int, *, eps: float, elementwise_affine: bool) ->
|
||||
|
||||
def apply_interleaved_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
|
||||
cos, sin = freqs
|
||||
x_real, x_imag = x.unflatten(2, (-1, 2)).unbind(-1) # [B, S, C // 2]
|
||||
# Concrete pair count instead of -1 keeps SDPA shape static under torch.compile.
|
||||
x_real, x_imag = x.unflatten(2, (x.shape[2] // 2, 2)).unbind(-1) # [B, S, C // 2]
|
||||
x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(2)
|
||||
out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype)
|
||||
return out
|
||||
|
||||
|
||||
def apply_split_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
|
||||
def apply_split_rotary_emb(
|
||||
x: torch.Tensor,
|
||||
freqs: tuple[torch.Tensor, torch.Tensor],
|
||||
*,
|
||||
head_dim: int,
|
||||
) -> torch.Tensor:
|
||||
# `head_dim` is plumbed in (not inferred via `-1`) so SDPA shape stays static under torch.compile.
|
||||
cos, sin = freqs
|
||||
|
||||
x_dtype = x.dtype
|
||||
@@ -83,7 +90,7 @@ def apply_split_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Ten
|
||||
# The cos/sin batch dim may only be broadcastable, so take batch size from x
|
||||
b = x.shape[0]
|
||||
_, h, t, _ = cos.shape
|
||||
x = x.reshape(b, t, h, -1).swapaxes(1, 2)
|
||||
x = x.reshape(b, t, h, head_dim).swapaxes(1, 2)
|
||||
needs_reshape = True
|
||||
|
||||
# Split last dim (2*r) into (d=2, r)
|
||||
@@ -110,7 +117,7 @@ def apply_split_rotary_emb(x: torch.Tensor, freqs: tuple[torch.Tensor, torch.Ten
|
||||
out = out.reshape(*out.shape[:-2], last)
|
||||
|
||||
if needs_reshape:
|
||||
out = out.swapaxes(1, 2).reshape(b, t, -1)
|
||||
out = out.swapaxes(1, 2).reshape(b, t, h * head_dim)
|
||||
|
||||
out = out.to(dtype=x_dtype)
|
||||
return out
|
||||
@@ -488,12 +495,14 @@ class LTX2AudioVideoAttnProcessor:
|
||||
key, key_rotary_emb if key_rotary_emb is not None else query_rotary_emb
|
||||
)
|
||||
elif attn.rope_type == "split":
|
||||
query = apply_split_rotary_emb(query, query_rotary_emb)
|
||||
key = apply_split_rotary_emb(key, key_rotary_emb if key_rotary_emb is not None else query_rotary_emb)
|
||||
query = apply_split_rotary_emb(query, query_rotary_emb, head_dim=attn.head_dim)
|
||||
key = apply_split_rotary_emb(
|
||||
key, key_rotary_emb if key_rotary_emb is not None else query_rotary_emb, head_dim=attn.head_dim
|
||||
)
|
||||
|
||||
query = query.unflatten(2, (attn.heads, -1))
|
||||
key = key.unflatten(2, (attn.heads, -1))
|
||||
value = value.unflatten(2, (attn.heads, -1))
|
||||
query = query.unflatten(2, (attn.heads, attn.head_dim))
|
||||
key = key.unflatten(2, (attn.heads, attn.head_dim))
|
||||
value = value.unflatten(2, (attn.heads, attn.head_dim))
|
||||
|
||||
attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None
|
||||
hidden_states = attn.attn(query, key, value, attn_metadata)
|
||||
@@ -502,7 +511,7 @@ class LTX2AudioVideoAttnProcessor:
|
||||
|
||||
# LTX-2.3: per-head gated attention
|
||||
if attn.to_gate_logits is not None:
|
||||
hidden_states = hidden_states.unflatten(2, (attn.heads, -1)) # [B, T, H, D]
|
||||
hidden_states = hidden_states.unflatten(2, (attn.heads, attn.head_dim)) # [B, T, H, D]
|
||||
# 2.0 * sigmoid so zero-init gates produce 1.0 (identity)
|
||||
gates = 2.0 * torch.sigmoid(gate_logits) # [B, T, H]
|
||||
hidden_states = hidden_states * gates.unsqueeze(-1)
|
||||
@@ -1391,9 +1400,11 @@ class LTX2AudioVideoRotaryPosEmbed(nn.Module):
|
||||
# Reshape freqs to be compatible with multi-head attention
|
||||
b = cos_freq.shape[0]
|
||||
t = cos_freq.shape[1]
|
||||
# Concrete per-head dim instead of -1 — see apply_split_rotary_emb
|
||||
r = self.dim // self.num_attention_heads // 2
|
||||
|
||||
cos_freq = cos_freq.reshape(b, t, self.num_attention_heads, -1)
|
||||
sin_freq = sin_freq.reshape(b, t, self.num_attention_heads, -1)
|
||||
cos_freq = cos_freq.reshape(b, t, self.num_attention_heads, r)
|
||||
sin_freq = sin_freq.reshape(b, t, self.num_attention_heads, r)
|
||||
|
||||
cos_freqs = torch.swapaxes(cos_freq, 1, 2) # (B,H,T,D//2)
|
||||
sin_freqs = torch.swapaxes(sin_freq, 1, 2) # (B,H,T,D//2)
|
||||
|
||||
@@ -157,6 +157,9 @@ class _VideoAudioScheduler:
|
||||
|
||||
|
||||
class LTX2Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin):
|
||||
# Audio is diffused jointly with video; warmup must size audio tokens.
|
||||
support_audio_output = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -115,6 +115,9 @@ class LTX23Pipeline(nn.Module, ProgressBarMixin):
|
||||
- CPU offloading: text encoder, connectors, VAE, vocoder stay on CPU
|
||||
"""
|
||||
|
||||
# Audio is diffused jointly with video; warmup must size audio tokens.
|
||||
support_audio_output = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in new issue
Block a user