[Model] Add Tencent AuK speech generation and editing (encoder + diffusion pipeline) (#7385)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
d3493384cf
commit
c111f3a2d6
28 files changed
+4986
No files matched your search
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""AuK diffusion-stage components: the flow transformer, the codec, the pipeline."""
|
||||
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
|
||||
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
|
||||
from vllm_omni.diffusion.models.auk.pipeline_auk import AuKPipeline, get_auk_post_process_func
|
||||
|
||||
__all__ = [
|
||||
"AuKPipeline",
|
||||
"AuKTransformer",
|
||||
"AuKVAE",
|
||||
"dit_state_dict",
|
||||
"get_auk_post_process_func",
|
||||
"sample_latents",
|
||||
]
|
||||
@@ -0,0 +1,677 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The AuK audio DiT as a self-contained inference module.
|
||||
|
||||
Two phases. ``num_layers`` double-stream blocks attend jointly over text and
|
||||
audio while keeping each stream's residual separate, then
|
||||
``num_single_layers`` single-stream blocks run over the concatenated
|
||||
``[text | ref audio | target audio]`` sequence. Only the target-audio slice is
|
||||
projected back to latent space, so the module predicts a flow-matching
|
||||
velocity for the target frames and :func:`sample_latents` integrates it.
|
||||
|
||||
Parameter names and shapes match the AuK reference backbone, so a released
|
||||
checkpoint loads with ``load_state_dict(strict=True)`` once the
|
||||
``transformer.`` prefix is stripped (see :func:`dit_state_dict`).
|
||||
|
||||
Deliberate differences from the reference, all inference-only:
|
||||
|
||||
* ``x_transformers`` and ``torchdiffeq`` are gone. Rotary embeddings are
|
||||
computed here (interleaved GPT-J pairs, no xpos scaling) and the ODE is an
|
||||
explicit Euler loop.
|
||||
* Dropout, activation checkpointing, the ``flash_attn`` backend switch and the
|
||||
zero-init of the modulation layers are training concerns and are dropped.
|
||||
* The timestep sinusoid is cast to the time MLP's weight dtype rather than to
|
||||
the timestep's own dtype, so an fp32 timestep works against half-precision
|
||||
weights outside autocast.
|
||||
* ``attn_mask_enabled`` defaults to ``True``, the value the released config
|
||||
sets, rather than the reference's ``False``.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
__all__ = ["AuKTransformer", "dit_state_dict", "sample_latents"]
|
||||
|
||||
|
||||
def _sdpa(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor:
|
||||
"""The single attention call site, so the backend can be swapped in one place.
|
||||
|
||||
``q``/``k``/``v`` are ``[B, H, N, D]``. ``mask`` is a boolean key-padding
|
||||
mask ``[B, K]`` in which ``True`` marks a valid key; it is broadcast to
|
||||
``[B, H, Q, K]`` only when given.
|
||||
"""
|
||||
attn_mask = None
|
||||
if mask is not None:
|
||||
attn_mask = mask[:, None, None, :].expand(q.shape[0], q.shape[1], q.shape[-2], k.shape[-2])
|
||||
return F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask, dropout_p=0.0, is_causal=False)
|
||||
|
||||
|
||||
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Rotate interleaved GPT-J pairs: ``(x1, x2) -> (-x2, x1)``."""
|
||||
pairs = x.reshape(*x.shape[:-1], -1, 2)
|
||||
x1, x2 = pairs.unbind(dim=-1)
|
||||
return torch.stack((-x2, x1), dim=-1).flatten(-2)
|
||||
|
||||
|
||||
def _apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply rotary frequencies ``[1, 1, N, D]`` to ``x`` ``[B, H, N, D]`` in fp32."""
|
||||
out_dtype = x.dtype
|
||||
x32 = x.float()
|
||||
out = x32 * freqs.cos() + _rotate_half(x32) * freqs.sin()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
class Rotary(nn.Module):
|
||||
"""Rotary position frequencies for one stream, evaluated in fp32.
|
||||
|
||||
The frequencies live in a buffer so that a released checkpoint's own copy
|
||||
of them loads with ``strict=True``. They are recomputed whenever the
|
||||
module has been cast below fp32: rounding either the frequencies or the
|
||||
integer positions to bf16 wrecks the phase at long positions.
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int, base: float = 10000.0) -> None:
|
||||
super().__init__()
|
||||
if dim % 2 != 0:
|
||||
raise ValueError(f"rotary dim must be even, got {dim}")
|
||||
self.dim = dim
|
||||
self.base = base
|
||||
self.register_buffer("inv_freq", self._frequencies(torch.device("cpu")))
|
||||
|
||||
def _frequencies(self, device: torch.device) -> torch.Tensor:
|
||||
exponents = torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim
|
||||
return 1.0 / (self.base**exponents)
|
||||
|
||||
def forward(self, seq_len: int) -> torch.Tensor:
|
||||
"""Return frequencies ``[1, 1, seq_len, dim]`` for positions ``0..seq_len-1``."""
|
||||
inv_freq = self.inv_freq
|
||||
if inv_freq.dtype != torch.float32:
|
||||
inv_freq = self._frequencies(inv_freq.device)
|
||||
pos = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32)
|
||||
freqs = torch.outer(pos, inv_freq)
|
||||
return torch.stack((freqs, freqs), dim=-1).flatten(-2)[None, None]
|
||||
|
||||
|
||||
class TimeEmbedding(nn.Module):
|
||||
"""Sinusoidal timestep features followed by a two-layer MLP."""
|
||||
|
||||
def __init__(self, dim: int, freq_embed_dim: int = 256) -> None:
|
||||
super().__init__()
|
||||
self.freq_embed_dim = freq_embed_dim
|
||||
self.time_mlp = nn.Sequential(nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim))
|
||||
|
||||
def forward(self, timestep: torch.Tensor, scale: float = 1000.0) -> torch.Tensor:
|
||||
half = self.freq_embed_dim // 2
|
||||
decay = math.log(10000) / (half - 1)
|
||||
freqs = torch.exp(torch.arange(half, device=timestep.device, dtype=torch.float32) * -decay)
|
||||
angles = scale * timestep.unsqueeze(1) * freqs.unsqueeze(0)
|
||||
hidden = torch.cat((angles.sin(), angles.cos()), dim=-1)
|
||||
return self.time_mlp(hidden.to(self.time_mlp[0].weight.dtype))
|
||||
|
||||
|
||||
class ConvPosEmbedding(nn.Module):
|
||||
"""Depthwise-grouped conv stack that adds local position information."""
|
||||
|
||||
def __init__(self, dim: int, kernel_size: int = 31, groups: int = 16) -> None:
|
||||
super().__init__()
|
||||
if kernel_size % 2 == 0:
|
||||
raise ValueError(f"kernel_size must be odd, got {kernel_size}")
|
||||
padding = kernel_size // 2
|
||||
self.conv1d = nn.Sequential(
|
||||
nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=padding),
|
||||
nn.Mish(),
|
||||
nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=padding),
|
||||
nn.Mish(),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
|
||||
keep = None if mask is None else mask.unsqueeze(1)
|
||||
x = x.transpose(1, 2)
|
||||
if keep is not None:
|
||||
x = x.masked_fill(~keep, 0.0)
|
||||
for layer in self.conv1d:
|
||||
x = layer(x)
|
||||
# Re-zero padding after each convolution so it cannot bleed into
|
||||
# valid frames through the kernel window.
|
||||
if keep is not None and isinstance(layer, nn.Conv1d):
|
||||
x = x.masked_fill(~keep, 0.0)
|
||||
return x.transpose(1, 2)
|
||||
|
||||
|
||||
class AudioEmbedding(nn.Module):
|
||||
"""Project audio latents to model width and add conv position information."""
|
||||
|
||||
def __init__(self, latent_dim: int, dim: int) -> None:
|
||||
super().__init__()
|
||||
self.linear = nn.Linear(latent_dim, dim)
|
||||
self.conv_pos_embed = ConvPosEmbedding(dim)
|
||||
|
||||
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
|
||||
x = self.linear(x)
|
||||
return self.conv_pos_embed(x, mask=mask) + x
|
||||
|
||||
|
||||
class AdaLayerNorm(nn.Module):
|
||||
"""Timestep-conditioned modulation for a block: six chunks from one projection."""
|
||||
|
||||
def __init__(self, dim: int) -> None:
|
||||
super().__init__()
|
||||
self.silu = nn.SiLU()
|
||||
self.linear = nn.Linear(dim, dim * 6)
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, emb: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.linear(self.silu(emb)).chunk(6, dim=1)
|
||||
x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
|
||||
return x, gate_msa, shift_mlp, scale_mlp, gate_mlp
|
||||
|
||||
|
||||
class AdaLayerNormFinal(nn.Module):
|
||||
"""Timestep-conditioned modulation before the output projection."""
|
||||
|
||||
def __init__(self, dim: int) -> None:
|
||||
super().__init__()
|
||||
self.silu = nn.SiLU()
|
||||
self.linear = nn.Linear(dim, dim * 2)
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
|
||||
def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor:
|
||||
scale, shift = self.linear(self.silu(emb)).chunk(2, dim=1)
|
||||
return self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
|
||||
|
||||
|
||||
class FeedForward(nn.Module):
|
||||
"""SwiGLU feed-forward with the gate projection fused into ``linear_in``."""
|
||||
|
||||
def __init__(self, dim: int, mult: float = 4.0) -> None:
|
||||
super().__init__()
|
||||
inner_dim = int(dim * mult)
|
||||
self.linear_in = nn.Linear(dim, inner_dim * 2, bias=False)
|
||||
self.linear_out = nn.Linear(inner_dim, dim, bias=False)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate, up = self.linear_in(x).chunk(2, dim=-1)
|
||||
return self.linear_out(F.silu(gate) * up)
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
"""Self-attention with per-head QK RMSNorm and rotary positions.
|
||||
|
||||
``to_out`` stays a ``ModuleList`` because the released checkpoints name the
|
||||
output projection ``to_out.0``; the reference's trailing dropout carries no
|
||||
parameters and is dropped.
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int, heads: int, dim_head: int, attn_mask_enabled: bool = True) -> None:
|
||||
super().__init__()
|
||||
self.heads = heads
|
||||
self.dim_head = dim_head
|
||||
self.attn_mask_enabled = attn_mask_enabled
|
||||
inner_dim = heads * dim_head
|
||||
self.to_qkv = nn.Linear(dim, 3 * inner_dim)
|
||||
self.q_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
|
||||
self.k_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
|
||||
self.to_out = nn.ModuleList([nn.Linear(inner_dim, dim)])
|
||||
|
||||
def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return x.transpose(1, 2).reshape(x.shape[0], -1, self.heads * self.dim_head)
|
||||
|
||||
def _qkv(
|
||||
self,
|
||||
packed: torch.Tensor,
|
||||
q_norm: nn.Module,
|
||||
k_norm: nn.Module,
|
||||
rope: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Split a packed QKV projection into heads, RMSNorm Q/K, then rotate."""
|
||||
q, k, v = (t.view(t.shape[0], -1, self.heads, self.dim_head).transpose(1, 2) for t in packed.chunk(3, dim=-1))
|
||||
q = q_norm(q)
|
||||
k = k_norm(k)
|
||||
if rope is not None:
|
||||
q = _apply_rope(q, rope)
|
||||
k = _apply_rope(k, rope)
|
||||
return q, k, v
|
||||
|
||||
def _project(self, x: torch.Tensor, rope: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return self._qkv(self.to_qkv(x), self.q_norm, self.k_norm, rope)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, mask: torch.Tensor | None = None, rope: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
q, k, v = self._project(x, rope)
|
||||
attn_mask = mask if self.attn_mask_enabled else None
|
||||
out = self._merge_heads(_sdpa(q, k, v, attn_mask).to(q.dtype))
|
||||
out = self.to_out[0](out)
|
||||
if mask is not None:
|
||||
out = out.masked_fill(~mask.unsqueeze(-1), 0.0)
|
||||
return out
|
||||
|
||||
|
||||
class JointAttention(Attention):
|
||||
"""Attention over the audio and text streams jointly, with separate projections."""
|
||||
|
||||
def __init__(self, dim: int, heads: int, dim_head: int, context_dim: int, attn_mask_enabled: bool = True) -> None:
|
||||
super().__init__(dim, heads, dim_head, attn_mask_enabled)
|
||||
inner_dim = heads * dim_head
|
||||
self.to_qkv_c = nn.Linear(context_dim, 3 * inner_dim)
|
||||
self.c_q_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
|
||||
self.c_k_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
|
||||
self.to_out_c = nn.Linear(inner_dim, context_dim)
|
||||
|
||||
def _project_context(self, c: torch.Tensor, c_rope: torch.Tensor | None) -> tuple[torch.Tensor, ...]:
|
||||
return self._qkv(self.to_qkv_c(c), self.c_q_norm, self.c_k_norm, c_rope)
|
||||
|
||||
def forward( # type: ignore[override] # joint attention takes the context stream too
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
c: torch.Tensor,
|
||||
mask: torch.Tensor | None = None,
|
||||
rope: torch.Tensor | None = None,
|
||||
c_rope: torch.Tensor | None = None,
|
||||
c_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
audio_len = x.shape[1]
|
||||
q, k, v = self._project(x, rope)
|
||||
c_q, c_k, c_v = self._project_context(c, c_rope)
|
||||
|
||||
q = torch.cat([q, c_q], dim=2)
|
||||
k = torch.cat([k, c_k], dim=2)
|
||||
v = torch.cat([v, c_v], dim=2)
|
||||
|
||||
joint_mask = None
|
||||
if self.attn_mask_enabled and mask is not None:
|
||||
if c_mask is not None:
|
||||
joint_mask = torch.cat([mask, c_mask], dim=1)
|
||||
else:
|
||||
joint_mask = F.pad(mask, (0, c.shape[1]), value=True)
|
||||
|
||||
out = self._merge_heads(_sdpa(q, k, v, joint_mask).to(q.dtype))
|
||||
x_out = self.to_out[0](out[:, :audio_len])
|
||||
c_out = self.to_out_c(out[:, audio_len:])
|
||||
|
||||
if mask is not None:
|
||||
x_out = x_out.masked_fill(~mask.unsqueeze(-1), 0.0)
|
||||
if c_mask is not None:
|
||||
c_out = c_out.masked_fill(~c_mask.unsqueeze(-1), 0.0)
|
||||
return x_out, c_out
|
||||
|
||||
|
||||
class DoubleBlock(nn.Module):
|
||||
"""MM-DiT block: joint attention, separate modulation and feed-forward per stream."""
|
||||
|
||||
def __init__(self, dim: int, heads: int, dim_head: int, ff_mult: float, attn_mask_enabled: bool) -> None:
|
||||
super().__init__()
|
||||
self.attn_norm_c = AdaLayerNorm(dim)
|
||||
self.attn_norm_x = AdaLayerNorm(dim)
|
||||
self.attn = JointAttention(
|
||||
dim=dim, heads=heads, dim_head=dim_head, context_dim=dim, attn_mask_enabled=attn_mask_enabled
|
||||
)
|
||||
self.ff_norm_c = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
self.ff_c = FeedForward(dim=dim, mult=ff_mult)
|
||||
self.ff_norm_x = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
self.ff_x = FeedForward(dim=dim, mult=ff_mult)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
c: torch.Tensor,
|
||||
t: torch.Tensor,
|
||||
mask: torch.Tensor | None,
|
||||
rope: torch.Tensor,
|
||||
c_rope: torch.Tensor,
|
||||
c_mask: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t)
|
||||
norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t)
|
||||
|
||||
x_attn, c_attn = self.attn(x=norm_x, c=norm_c, mask=mask, rope=rope, c_rope=c_rope, c_mask=c_mask)
|
||||
|
||||
c = c + c_gate_msa[:, None] * c_attn
|
||||
norm_c = self.ff_norm_c(c) * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None]
|
||||
c = c + c_gate_mlp[:, None] * self.ff_c(norm_c)
|
||||
|
||||
x = x + x_gate_msa[:, None] * x_attn
|
||||
norm_x = self.ff_norm_x(x) * (1 + x_scale_mlp[:, None]) + x_shift_mlp[:, None]
|
||||
x = x + x_gate_mlp[:, None] * self.ff_x(norm_x)
|
||||
return c, x
|
||||
|
||||
|
||||
class SingleBlock(nn.Module):
|
||||
"""DiT block over the concatenated text and audio sequence."""
|
||||
|
||||
def __init__(self, dim: int, heads: int, dim_head: int, ff_mult: float, attn_mask_enabled: bool) -> None:
|
||||
super().__init__()
|
||||
self.attn_norm = AdaLayerNorm(dim)
|
||||
self.attn = Attention(dim=dim, heads=heads, dim_head=dim_head, attn_mask_enabled=attn_mask_enabled)
|
||||
self.ff_norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
self.ff = FeedForward(dim=dim, mult=ff_mult)
|
||||
|
||||
def forward(self, x: torch.Tensor, t: torch.Tensor, mask: torch.Tensor | None, rope: torch.Tensor) -> torch.Tensor:
|
||||
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t)
|
||||
x = x + gate_msa[:, None] * self.attn(x=norm, mask=mask, rope=rope)
|
||||
norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
|
||||
return x + gate_mlp[:, None] * self.ff(norm)
|
||||
|
||||
|
||||
class AuKTransformer(nn.Module):
|
||||
"""Flow-matching velocity predictor for AuK audio latents.
|
||||
|
||||
Args:
|
||||
dim: Model width.
|
||||
heads: Attention heads.
|
||||
dim_head: Per-head width.
|
||||
ff_mult: Feed-forward expansion factor.
|
||||
latent_dim: Audio VAE latent channels, the input and output width.
|
||||
text_hidden_dim: Width of the pre-encoded text hidden states.
|
||||
num_layers: Double-stream (joint attention) block count.
|
||||
num_single_layers: Single-stream block count.
|
||||
attn_mask_enabled: Apply padding masks inside attention. When ``False``
|
||||
attention runs unmasked, though padded outputs are still zeroed,
|
||||
which is what the reference does with the flag off.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dim: int,
|
||||
heads: int,
|
||||
dim_head: int,
|
||||
ff_mult: float,
|
||||
latent_dim: int,
|
||||
text_hidden_dim: int,
|
||||
num_layers: int,
|
||||
num_single_layers: int,
|
||||
attn_mask_enabled: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.latent_dim = latent_dim
|
||||
self.text_cond: torch.Tensor | None = None
|
||||
self.text_uncond: torch.Tensor | None = None
|
||||
|
||||
self.time_embed = TimeEmbedding(dim)
|
||||
self.txt_norm = nn.RMSNorm(dim, elementwise_affine=True)
|
||||
self.txt_proj = nn.Linear(text_hidden_dim, dim)
|
||||
self.audio_embed = AudioEmbedding(latent_dim, dim)
|
||||
self.rotary_embed = Rotary(dim_head)
|
||||
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
DoubleBlock(
|
||||
dim=dim, heads=heads, dim_head=dim_head, ff_mult=ff_mult, attn_mask_enabled=attn_mask_enabled
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.single_transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
SingleBlock(
|
||||
dim=dim, heads=heads, dim_head=dim_head, ff_mult=ff_mult, attn_mask_enabled=attn_mask_enabled
|
||||
)
|
||||
for _ in range(num_single_layers)
|
||||
]
|
||||
)
|
||||
|
||||
self.norm_out = AdaLayerNormFinal(dim)
|
||||
self.proj_out = nn.Linear(dim, latent_dim)
|
||||
|
||||
def project_text(self, text: torch.Tensor) -> torch.Tensor:
|
||||
"""Project LLM hidden states ``[B, nt, text_hidden_dim]`` to model width."""
|
||||
return self.txt_norm(self.txt_proj(text))
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Drop the cached text projection. Call this when the conditioning changes."""
|
||||
self.text_cond = None
|
||||
self.text_uncond = None
|
||||
|
||||
def _embed_audio(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
ref: torch.Tensor | None,
|
||||
drop_audio_cond: bool,
|
||||
mask: torch.Tensor | None,
|
||||
ref_mask: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, int]:
|
||||
"""Embed the target latents, prepending the reference prompt when present.
|
||||
|
||||
Returns the ``[ref | target]`` sequence, its padding mask and the
|
||||
reference length, which the caller uses to slice the target back out.
|
||||
"""
|
||||
if ref is not None and ref.shape[1] == 0:
|
||||
ref = None
|
||||
|
||||
target = self.audio_embed(x, mask=mask)
|
||||
if ref is None:
|
||||
return target, mask, 0
|
||||
|
||||
if drop_audio_cond:
|
||||
ref = torch.zeros_like(ref)
|
||||
prompt = self.audio_embed(ref, mask=ref_mask)
|
||||
audio = torch.cat([prompt, target], dim=1)
|
||||
prompt_len = prompt.shape[1]
|
||||
|
||||
audio_mask = None
|
||||
if mask is not None or ref_mask is not None:
|
||||
if mask is None:
|
||||
mask = x.new_ones(x.shape[:2], dtype=torch.bool)
|
||||
if ref_mask is None:
|
||||
ref_mask = ref.new_ones(ref.shape[:2], dtype=torch.bool)
|
||||
audio_mask = torch.cat([ref_mask, mask], dim=1)
|
||||
return audio, audio_mask, prompt_len
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
text: torch.Tensor,
|
||||
time: torch.Tensor,
|
||||
*,
|
||||
mask: torch.Tensor | None = None,
|
||||
c_mask: torch.Tensor | None = None,
|
||||
ref: torch.Tensor | None = None,
|
||||
ref_mask: torch.Tensor | None = None,
|
||||
drop_audio_cond: bool = False,
|
||||
drop_text: bool = False,
|
||||
cfg_infer: bool = False,
|
||||
cache: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Predict the flow-matching velocity for the target frames of ``x``.
|
||||
|
||||
Args:
|
||||
x: Noised target latents ``[B, n, latent_dim]``.
|
||||
text: Pre-encoded text hidden states ``[B, nt, text_hidden_dim]``.
|
||||
time: Flow-matching timestep, a scalar or ``[B]``.
|
||||
mask: Target padding mask ``[B, n]``, ``True`` where valid.
|
||||
c_mask: Text padding mask ``[B, nt]``. Inferred from all-zero text
|
||||
rows when omitted.
|
||||
ref: Reference prompt latents ``[B, np, latent_dim]``, prepended to
|
||||
the audio sequence. A zero-length ``ref`` counts as absent.
|
||||
ref_mask: Reference padding mask ``[B, np]``.
|
||||
drop_audio_cond: Zero the reference prompt, for the CFG uncond branch.
|
||||
drop_text: Zero the projected text, for the CFG uncond branch.
|
||||
cfg_infer: Run the cond and uncond branches as one batch of ``2B``,
|
||||
cond first. ``drop_audio_cond`` and ``drop_text`` are then
|
||||
driven per branch and ignored.
|
||||
cache: Reuse the projected text across calls, for an ODE loop over
|
||||
fixed conditioning. Only the ``cfg_infer`` path caches, as in
|
||||
the reference. Call :meth:`clear_cache` when the text changes.
|
||||
|
||||
Returns:
|
||||
Velocity ``[B, n, latent_dim]``, or ``[2B, n, latent_dim]`` under
|
||||
``cfg_infer``.
|
||||
"""
|
||||
batch = x.shape[0]
|
||||
if time.ndim == 0:
|
||||
time = time.repeat(batch)
|
||||
t = self.time_embed(time)
|
||||
|
||||
if c_mask is None:
|
||||
# Padding rows are all-zero in the text encoder's output.
|
||||
c_mask = text.abs().sum(-1) > 0
|
||||
|
||||
if cfg_infer:
|
||||
if cache and self.text_cond is not None:
|
||||
c_cond, c_uncond = self.text_cond, self.text_uncond
|
||||
else:
|
||||
c_cond = self.project_text(text)
|
||||
# The uncond branch drops the text, which is exactly a zeroed projection.
|
||||
c_uncond = torch.zeros_like(c_cond)
|
||||
if cache:
|
||||
self.text_cond, self.text_uncond = c_cond, c_uncond
|
||||
x_cond, mask_cond, prompt_len = self._embed_audio(x, ref, False, mask, ref_mask)
|
||||
x_uncond, mask_uncond, _ = self._embed_audio(x, ref, True, mask, ref_mask)
|
||||
|
||||
x = torch.cat((x_cond, x_uncond), dim=0)
|
||||
c = torch.cat((c_cond, c_uncond), dim=0)
|
||||
t = torch.cat((t, t), dim=0)
|
||||
c_mask = torch.cat((c_mask, c_mask), dim=0)
|
||||
audio_mask = None
|
||||
if mask_cond is not None and mask_uncond is not None:
|
||||
audio_mask = torch.cat((mask_cond, mask_uncond), dim=0)
|
||||
else:
|
||||
c = self.project_text(text)
|
||||
if drop_text:
|
||||
c = torch.zeros_like(c)
|
||||
x, audio_mask, prompt_len = self._embed_audio(x, ref, drop_audio_cond, mask, ref_mask)
|
||||
|
||||
seq_len = x.shape[1]
|
||||
text_len = c.shape[1]
|
||||
rope_audio = self.rotary_embed(seq_len)
|
||||
rope_text = self.rotary_embed(text_len)
|
||||
|
||||
for block in self.transformer_blocks:
|
||||
c, x = block(x, c, t, mask=audio_mask, rope=rope_audio, c_rope=rope_text, c_mask=c_mask)
|
||||
|
||||
x = torch.cat([c, x], dim=1)
|
||||
rope = self.rotary_embed(text_len + seq_len)
|
||||
single_mask = None if audio_mask is None else torch.cat([c_mask, audio_mask], dim=1)
|
||||
|
||||
for block in self.single_transformer_blocks:
|
||||
x = block(x, t, mask=single_mask, rope=rope)
|
||||
|
||||
x = x[:, text_len + prompt_len :]
|
||||
return self.proj_out(self.norm_out(x, t))
|
||||
|
||||
|
||||
def dit_state_dict(
|
||||
checkpoint: Mapping[str, torch.Tensor] | Iterable[tuple[str, torch.Tensor]],
|
||||
prefix: str = "transformer.",
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Select the DiT tensors out of a full AuK checkpoint and strip ``prefix``.
|
||||
|
||||
Released checkpoints wrap the backbone under ``transformer.`` alongside the
|
||||
text-encoder layer-fusion parameters, which this module does not own.
|
||||
"""
|
||||
items = checkpoint.items() if isinstance(checkpoint, Mapping) else checkpoint
|
||||
return {name[len(prefix) :]: tensor for name, tensor in items if name.startswith(prefix)}
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def sample_latents(
|
||||
dit: AuKTransformer,
|
||||
*,
|
||||
text: torch.Tensor,
|
||||
c_mask: torch.Tensor | None,
|
||||
ref: torch.Tensor,
|
||||
ref_mask: torch.Tensor | None,
|
||||
gen_frames: int,
|
||||
nfe: int = 32,
|
||||
cfg_strength: float = 1.0,
|
||||
sway_sampling_coef: float | None = None,
|
||||
t_grid: list[float] | None = None,
|
||||
seed: int | None = None,
|
||||
generator: torch.Generator | None = None,
|
||||
latent_dim: int | None = None,
|
||||
device: torch.device | str | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Integrate the flow from noise to audio latents with explicit Euler steps.
|
||||
|
||||
Single-request only: the batch dimension carries the CFG branches, not
|
||||
separate prompts, so no target padding mask is needed.
|
||||
|
||||
Args:
|
||||
dit: The velocity model.
|
||||
text: Pre-encoded text hidden states ``[1, nt, text_hidden_dim]``.
|
||||
c_mask: Text padding mask ``[1, nt]``.
|
||||
ref: Reference prompt latents ``[1, np, latent_dim]``.
|
||||
ref_mask: Reference padding mask ``[1, np]``.
|
||||
gen_frames: Target latent frames to generate.
|
||||
nfe: Euler steps, ignored when ``t_grid`` is given.
|
||||
cfg_strength: Classifier-free guidance weight. Below ``1e-5`` the
|
||||
uncond branch is skipped entirely.
|
||||
sway_sampling_coef: Reshapes the uniform time grid towards ``t=0``.
|
||||
t_grid: Explicit timesteps, overriding ``nfe`` and the sway reshape.
|
||||
seed: Seeds the global RNG before drawing the noise. Ignored when
|
||||
``generator`` is given.
|
||||
generator: Draws the initial noise from its own RNG, leaving the global
|
||||
one untouched.
|
||||
latent_dim: Latent channels; defaults to the model's.
|
||||
device: Device for the noise; defaults to the reference's.
|
||||
dtype: Dtype for the noise and the Euler accumulator; defaults to the
|
||||
reference's, which is the reference implementation's rule. Pass
|
||||
``torch.float32`` explicitly when serving half-precision weights.
|
||||
Drawing the noise at bf16 instead of fp32 moves a 32-step CFG
|
||||
trajectory by 0.025 relative MSE, roughly 17 times what bf16
|
||||
weights themselves cost, because the draw consumes the generator
|
||||
differently and starts the ODE somewhere else. Keeping it fp32
|
||||
costs a few tens of KB and no measurable time.
|
||||
|
||||
Returns:
|
||||
The target latents at ``t=1``, ``[1, gen_frames, latent_dim]``.
|
||||
"""
|
||||
device = torch.device(device) if device is not None else ref.device
|
||||
dtype = dtype if dtype is not None else ref.dtype
|
||||
latent_dim = latent_dim if latent_dim is not None else dit.latent_dim
|
||||
|
||||
if generator is None:
|
||||
if seed is not None:
|
||||
torch.manual_seed(seed)
|
||||
x = torch.randn(gen_frames, latent_dim, device=device, dtype=dtype).unsqueeze(0)
|
||||
else:
|
||||
x = (
|
||||
torch.randn(gen_frames, latent_dim, generator=generator, device=generator.device, dtype=dtype)
|
||||
.to(device)
|
||||
.unsqueeze(0)
|
||||
)
|
||||
|
||||
if t_grid is not None:
|
||||
t = torch.tensor(t_grid, device=device, dtype=torch.float32)
|
||||
else:
|
||||
t = torch.linspace(0, 1, nfe + 1, device=device, dtype=torch.float32)
|
||||
if sway_sampling_coef is not None:
|
||||
t = t + sway_sampling_coef * (torch.cos(math.pi / 2 * t) - 1 + t)
|
||||
|
||||
guided = cfg_strength >= 1e-5
|
||||
if t.numel() < 2 or not bool(torch.all(t[1:] > t[:-1])):
|
||||
raise ValueError(
|
||||
f"AuK sampling needs a strictly increasing time grid with at least two points; got {t.tolist()}"
|
||||
)
|
||||
try:
|
||||
for i in range(t.shape[0] - 1):
|
||||
if guided:
|
||||
pred = dit(
|
||||
x,
|
||||
text,
|
||||
t[i],
|
||||
c_mask=c_mask,
|
||||
ref=ref,
|
||||
ref_mask=ref_mask,
|
||||
cfg_infer=True,
|
||||
cache=True,
|
||||
)
|
||||
v_cond, v_uncond = pred.chunk(2, dim=0)
|
||||
v = v_cond + (v_cond - v_uncond) * cfg_strength
|
||||
else:
|
||||
v = dit(x, text, t[i], c_mask=c_mask, ref=ref, ref_mask=ref_mask)
|
||||
x = x + (t[i + 1] - t[i]) * v
|
||||
finally:
|
||||
# The cached text projections belong to this request only; a failed
|
||||
# step must not leak them into the next one.
|
||||
dit.clear_cache()
|
||||
return x
|
||||
@@ -0,0 +1,505 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
#
|
||||
# Adapted from the AuK reference release (MIT), which in turn adapts NVIDIA
|
||||
# BigVGAN (MIT), jik876/hifi-gan (MIT), junjun3518/alias-free-torch
|
||||
# (Apache-2.0), EdwardDixon/snake (MIT) and adefossez/julius (MIT).
|
||||
"""BigVGAN-flow VAE for AuK: 24 kHz mono waveform <-> 50 Hz, 64-channel latents.
|
||||
|
||||
Inference only. The reference module also carries a residual coupling flow, a KL
|
||||
term and a discriminator; all three are training-only, so the released
|
||||
``flow.*`` tensors are skipped (see :meth:`AuKVAE.load_weights`).
|
||||
|
||||
Parameter and buffer names follow the reference release, so the published
|
||||
``vae.safetensors`` loads without a key map. That is also why the convolutions
|
||||
are built with the legacy ``weight_norm`` parametrization (``weight_g`` /
|
||||
``weight_v``): :meth:`remove_weight_norm` folds them into plain weights once
|
||||
the checkpoint is in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import math
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from safetensors.torch import load_file
|
||||
from torch import nn
|
||||
from torch.nn.utils import remove_weight_norm as _fold_weight_norm
|
||||
from torch.nn.utils import weight_norm as _apply_weight_norm
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["AuKVAE"]
|
||||
|
||||
# Checkpoint prefix owned by the training-only coupling flow.
|
||||
TRAINING_ONLY_PREFIX = "flow."
|
||||
|
||||
|
||||
def _weight_norm(module: nn.Module) -> nn.Module:
|
||||
"""Legacy ``weight_g``/``weight_v`` split, which is how the release stores these tensors."""
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings("ignore", message=".*weight_norm.*deprecated.*", category=FutureWarning)
|
||||
return _apply_weight_norm(module)
|
||||
|
||||
|
||||
def _same_padding(kernel_size: int, dilation: int = 1) -> int:
|
||||
return int((kernel_size * dilation - dilation) / 2)
|
||||
|
||||
|
||||
def _kaiser_sinc_filter(cutoff: float, half_width: float, kernel_size: int) -> torch.Tensor:
|
||||
"""Kaiser-windowed sinc low-pass, returned as a ``[1, 1, kernel_size]`` conv kernel."""
|
||||
half_size = kernel_size // 2
|
||||
attenuation = 2.285 * (half_size - 1) * math.pi * (4 * half_width) + 7.95
|
||||
if attenuation > 50.0:
|
||||
beta = 0.1102 * (attenuation - 8.7)
|
||||
elif attenuation >= 21.0:
|
||||
beta = 0.5842 * (attenuation - 21.0) ** 0.4 + 0.07886 * (attenuation - 21.0)
|
||||
else:
|
||||
beta = 0.0
|
||||
window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)
|
||||
if kernel_size % 2 == 0:
|
||||
time = torch.arange(-half_size, half_size) + 0.5
|
||||
else:
|
||||
time = torch.arange(kernel_size) - half_size
|
||||
if cutoff == 0:
|
||||
return torch.zeros_like(time).view(1, 1, kernel_size)
|
||||
taps = 2 * cutoff * window * torch.sinc(2 * cutoff * time)
|
||||
# Normalize to unit sum so a constant input passes through unscaled.
|
||||
return (taps / taps.sum()).view(1, 1, kernel_size)
|
||||
|
||||
|
||||
class Conv(nn.Conv1d):
|
||||
"""1-D convolution over ``[B, C, T]``; ``causal`` swaps centred padding for a left pad."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int,
|
||||
out_channels: int,
|
||||
kernel_size: int = 1,
|
||||
stride: int = 1,
|
||||
dilation: int = 1,
|
||||
bias: bool = True,
|
||||
causal: bool = False,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
in_channels,
|
||||
out_channels,
|
||||
kernel_size,
|
||||
stride=stride,
|
||||
padding=0 if causal else _same_padding(kernel_size, dilation),
|
||||
dilation=dilation,
|
||||
bias=bias,
|
||||
)
|
||||
self.causal = causal
|
||||
self.left_padding = dilation * (kernel_size - 1) if causal else 0
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.causal:
|
||||
x = F.pad(x, (self.left_padding, 0))
|
||||
return super().forward(x)
|
||||
|
||||
|
||||
class ConvTranspose(nn.ConvTranspose1d):
|
||||
"""Transposed 1-D convolution; ``causal`` drops the trailing ``stride`` samples instead of padding."""
|
||||
|
||||
def __init__(
|
||||
self, in_channels: int, out_channels: int, kernel_size: int, stride: int, causal: bool = False
|
||||
) -> None:
|
||||
if causal and kernel_size != 2 * stride:
|
||||
raise ValueError(f"causal ConvTranspose needs kernel_size == 2 * stride, got {kernel_size} and {stride}")
|
||||
super().__init__(
|
||||
in_channels,
|
||||
out_channels,
|
||||
kernel_size,
|
||||
stride=stride,
|
||||
padding=0 if causal else (kernel_size - stride) // 2,
|
||||
)
|
||||
self.causal = causal
|
||||
self.trim = stride
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = super().forward(x)
|
||||
return x[:, :, : -self.trim] if self.causal else x
|
||||
|
||||
|
||||
class NormConv(nn.Module):
|
||||
"""Weight-normalized conv held under ``layer``; the attribute name is load-bearing for the checkpoint."""
|
||||
|
||||
def __init__(self, in_channels: int, out_channels: int, kernel_size: int = 1, stride: int = 1) -> None:
|
||||
super().__init__()
|
||||
self.layer = _weight_norm(
|
||||
nn.Conv1d(in_channels, out_channels, kernel_size, stride=stride, padding=(kernel_size - 1) // 2)
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.layer(x)
|
||||
|
||||
|
||||
class ResStack(nn.Module):
|
||||
"""Stack of dilated residual conv pairs. The leaky slope is torch's 0.01 default, as upstream."""
|
||||
|
||||
def __init__(self, channels: int, kernel_size: int = 3, base: int = 3, nums: int = 4) -> None:
|
||||
super().__init__()
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
nn.Sequential(
|
||||
nn.LeakyReLU(),
|
||||
_weight_norm(nn.Conv1d(channels, channels, kernel_size, dilation=base**i, padding=base**i)),
|
||||
nn.LeakyReLU(),
|
||||
_weight_norm(nn.Conv1d(channels, channels, kernel_size, dilation=1, padding=1)),
|
||||
)
|
||||
for i in range(nums)
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
for layer in self.layers:
|
||||
x = x + layer(x)
|
||||
return x
|
||||
|
||||
|
||||
class Encoder(nn.Module):
|
||||
"""Strided conv encoder over ``[B, 1, T]``, emitting ``[B, 2 * latent_dim, T / hop]`` mean/log-std pairs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
latent_dim: int,
|
||||
channels: tuple[int, ...],
|
||||
down_sample_factors: tuple[int, ...],
|
||||
in_channels: int = 1,
|
||||
proj_kernel_size: int = 3,
|
||||
stack_kernel_size: int = 3,
|
||||
stack_dilation_base: int = 2,
|
||||
stacks: int = 6,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
layers: list[nn.Module] = [
|
||||
NormConv(in_channels, channels[0], kernel_size=proj_kernel_size),
|
||||
nn.LeakyReLU(0.2, True),
|
||||
]
|
||||
for in_c, out_c, factor in zip(channels[:-1], channels[1:], down_sample_factors, strict=True):
|
||||
layers += [
|
||||
NormConv(in_c, out_c, kernel_size=factor * 2, stride=factor),
|
||||
ResStack(out_c, stack_kernel_size, stack_dilation_base, stacks),
|
||||
nn.LeakyReLU(0.2, True),
|
||||
]
|
||||
layers.append(NormConv(channels[-1], latent_dim * 2, kernel_size=proj_kernel_size))
|
||||
self.generator = nn.Sequential(*layers)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.generator(x)
|
||||
|
||||
|
||||
class SnakeBeta(nn.Module):
|
||||
"""``x + sin^2(alpha * x) / beta`` with per-channel learned alpha and beta (log-scale in AuK)."""
|
||||
|
||||
def __init__(self, channels: int, alpha_logscale: bool = False) -> None:
|
||||
super().__init__()
|
||||
self.alpha_logscale = alpha_logscale
|
||||
init = torch.zeros(channels) if alpha_logscale else torch.ones(channels)
|
||||
self.alpha = nn.Parameter(init.clone())
|
||||
self.beta = nn.Parameter(init.clone())
|
||||
self.eps = 1e-9
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
|
||||
beta = self.beta.unsqueeze(0).unsqueeze(-1)
|
||||
if self.alpha_logscale:
|
||||
alpha = torch.exp(alpha)
|
||||
beta = torch.exp(beta)
|
||||
return x + (1.0 / (beta + self.eps)) * torch.sin(x * alpha).pow(2)
|
||||
|
||||
|
||||
class LowPass(nn.Module):
|
||||
"""Fixed FIR low-pass with replicate padding, applied per channel."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cutoff: float = 0.5,
|
||||
half_width: float = 0.6,
|
||||
stride: int = 1,
|
||||
kernel_size: int = 12,
|
||||
causal: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if not 0.0 <= cutoff <= 0.5:
|
||||
raise ValueError(f"cutoff must lie in [0, 0.5], got {cutoff}")
|
||||
if causal:
|
||||
self.pad_left, self.pad_right = kernel_size - 1, 0
|
||||
else:
|
||||
even = kernel_size % 2 == 0
|
||||
self.pad_left = kernel_size // 2 - int(even)
|
||||
self.pad_right = kernel_size // 2
|
||||
self.stride = stride
|
||||
self.register_buffer("filter", _kaiser_sinc_filter(cutoff, half_width, kernel_size))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
channels = x.size(1)
|
||||
x = F.pad(x, (self.pad_left, self.pad_right), mode="replicate")
|
||||
return F.conv1d(x, self.filter.expand(channels, -1, -1), stride=self.stride, groups=channels)
|
||||
|
||||
|
||||
class Upsample(nn.Module):
|
||||
"""Band-limited interpolation by ``ratio``. Always non-causal, matching how AuK builds it."""
|
||||
|
||||
def __init__(self, ratio: int = 2, kernel_size: int = 12) -> None:
|
||||
super().__init__()
|
||||
self.ratio = ratio
|
||||
self.pad = kernel_size // ratio - 1
|
||||
self.pad_left = self.pad * ratio + (kernel_size - ratio) // 2
|
||||
self.pad_right = self.pad * ratio + (kernel_size - ratio + 1) // 2
|
||||
self.register_buffer("filter", _kaiser_sinc_filter(0.5 / ratio, 0.6 / ratio, kernel_size))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
channels = x.size(1)
|
||||
x = F.pad(x, (self.pad, self.pad), mode="replicate")
|
||||
x = self.ratio * F.conv_transpose1d(x, self.filter.expand(channels, -1, -1), stride=self.ratio, groups=channels)
|
||||
return x[..., self.pad_left : -self.pad_right]
|
||||
|
||||
|
||||
class Downsample(nn.Module):
|
||||
"""Decimation by ``ratio`` behind a low-pass; ``lowpass`` holds the filter buffer."""
|
||||
|
||||
def __init__(self, ratio: int = 2, kernel_size: int = 12, causal: bool = False) -> None:
|
||||
super().__init__()
|
||||
self.lowpass = LowPass(
|
||||
cutoff=0.5 / ratio, half_width=0.6 / ratio, stride=ratio, kernel_size=kernel_size, causal=causal
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.lowpass(x)
|
||||
|
||||
|
||||
class AliasFreeActivation(nn.Module):
|
||||
"""Oversample, apply the pointwise activation, decimate, so snake harmonics do not fold back."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
activation: nn.Module,
|
||||
up_ratio: int = 2,
|
||||
down_ratio: int = 2,
|
||||
up_kernel_size: int = 12,
|
||||
down_kernel_size: int = 12,
|
||||
causal: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.act = activation
|
||||
self.upsample = Upsample(up_ratio, up_kernel_size)
|
||||
self.downsample = Downsample(down_ratio, down_kernel_size, causal=causal)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.downsample(self.act(self.upsample(x)))
|
||||
|
||||
|
||||
class AmpBlock(nn.Module):
|
||||
"""Anti-aliased multi-periodicity residual block: alternating alias-free snake and dilated convs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
channels: int,
|
||||
kernel_size: int,
|
||||
dilations: tuple[int, ...],
|
||||
snake_logscale: bool = True,
|
||||
causal: bool = True,
|
||||
act_causal: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.convs1 = nn.ModuleList(
|
||||
[_weight_norm(Conv(channels, channels, kernel_size, 1, dilation=d, causal=causal)) for d in dilations]
|
||||
)
|
||||
self.convs2 = nn.ModuleList(
|
||||
[_weight_norm(Conv(channels, channels, kernel_size, 1, dilation=1, causal=causal)) for _ in dilations]
|
||||
)
|
||||
self.activations = nn.ModuleList(
|
||||
[
|
||||
AliasFreeActivation(SnakeBeta(channels, alpha_logscale=snake_logscale), causal=act_causal)
|
||||
for _ in range(len(self.convs1) + len(self.convs2))
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
acts1, acts2 = self.activations[::2], self.activations[1::2]
|
||||
for conv1, conv2, act1, act2 in zip(self.convs1, self.convs2, acts1, acts2, strict=True):
|
||||
x = conv2(act2(conv1(act1(x)))) + x
|
||||
return x
|
||||
|
||||
|
||||
class AuKVAE(nn.Module):
|
||||
"""AuK's audio codec: :meth:`encode` waveform to normalized latents, :meth:`decode` back.
|
||||
|
||||
Build from the released ``model.vae.model_init_kwargs`` with :meth:`from_config`, then
|
||||
:meth:`load_weights`. Latents are normalized by the checkpoint's global statistics, which is
|
||||
the space the rectified-flow transformer works in.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
upsample_rates: tuple[int, ...] = (5, 4, 3, 2, 2, 2),
|
||||
upsample_kernel_sizes: tuple[int, ...] = (10, 8, 6, 4, 4, 4),
|
||||
upsample_initial_channel: int = 1536,
|
||||
resblock_kernel_sizes: tuple[int, ...] = (3, 7, 11),
|
||||
resblock_dilation_sizes: tuple[tuple[int, ...], ...] = ((1, 3, 5), (1, 3, 5), (1, 3, 5)),
|
||||
downsample_rates: tuple[int, ...] = (2, 2, 2, 3, 4, 5),
|
||||
downsample_channels: tuple[int, ...] = (12, 24, 48, 96, 192, 384, 768),
|
||||
snake_logscale: bool = True,
|
||||
latent_dim: int = 64,
|
||||
use_vae: bool = True,
|
||||
causal: bool = True,
|
||||
act_causal: bool = True,
|
||||
sample_rate: int = 24000,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if not use_vae:
|
||||
raise ValueError("AuKVAE requires use_vae=True: the encoder head emits mean/log-std pairs")
|
||||
self.latent_dim = latent_dim
|
||||
self.sample_rate = sample_rate
|
||||
self.hop_size = math.prod(downsample_rates)
|
||||
self.num_kernels = len(resblock_kernel_sizes)
|
||||
self.num_upsamples = len(upsample_rates)
|
||||
self._norm_folded = False
|
||||
|
||||
self.register_buffer("global_mean", torch.zeros(latent_dim, dtype=torch.float32))
|
||||
self.register_buffer("global_log_std", torch.ones(latent_dim, dtype=torch.float32))
|
||||
|
||||
self.audio_encoder = Encoder(
|
||||
latent_dim=latent_dim,
|
||||
channels=tuple(downsample_channels),
|
||||
down_sample_factors=tuple(downsample_rates),
|
||||
)
|
||||
|
||||
self.conv_pre = _weight_norm(Conv(latent_dim, upsample_initial_channel, 7, 1, causal=False))
|
||||
self.ups = nn.ModuleList()
|
||||
for i, (rate, kernel) in enumerate(zip(upsample_rates, upsample_kernel_sizes, strict=True)):
|
||||
# One-element inner list: the checkpoint keys are ``ups.<i>.0.*``.
|
||||
self.ups.append(
|
||||
nn.ModuleList(
|
||||
[
|
||||
_weight_norm(
|
||||
ConvTranspose(
|
||||
upsample_initial_channel // (2**i),
|
||||
upsample_initial_channel // (2 ** (i + 1)),
|
||||
kernel,
|
||||
rate,
|
||||
causal=causal,
|
||||
)
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
self.resblocks = nn.ModuleList()
|
||||
channels = upsample_initial_channel
|
||||
for i in range(self.num_upsamples):
|
||||
channels = upsample_initial_channel // (2 ** (i + 1))
|
||||
for kernel, dilations in zip(resblock_kernel_sizes, resblock_dilation_sizes, strict=True):
|
||||
self.resblocks.append(
|
||||
AmpBlock(
|
||||
channels,
|
||||
kernel,
|
||||
tuple(dilations),
|
||||
snake_logscale=snake_logscale,
|
||||
causal=causal,
|
||||
act_causal=act_causal,
|
||||
)
|
||||
)
|
||||
|
||||
self.activation_post = AliasFreeActivation(
|
||||
SnakeBeta(channels, alpha_logscale=snake_logscale), causal=act_causal
|
||||
)
|
||||
self.conv_post = _weight_norm(Conv(channels, 1, 7, 1, causal=causal, bias=False))
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, cfg: dict) -> AuKVAE:
|
||||
"""Build from the released ``model_init_kwargs`` mapping, ignoring training-only entries."""
|
||||
accepted = set(inspect.signature(cls.__init__).parameters) - {"self"}
|
||||
return cls(**{key: value for key, value in cfg.items() if key in accepted})
|
||||
|
||||
def load_weights(
|
||||
self, path: str | Path, *, device: str | torch.device | None = None, fold_weight_norm: bool = True
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Load a released ``vae.safetensors``, then fold weight norm away. Returns (missing, unexpected).
|
||||
|
||||
``device`` defaults to where the module already lives, which is the placement that matters:
|
||||
the fold reduces ``weight_v`` to a norm, and a CPU reduction lands up to one ULP away from the
|
||||
accelerator's. The encoder amplifies its activations roughly fiftyfold, so that one ULP grows
|
||||
into a ~4e-4 relative drift by the latent head. Move the module first, then load.
|
||||
"""
|
||||
target = self.global_mean.device if device is None else torch.device(device)
|
||||
missing, unexpected = self.load_state_dict(load_file(str(path), device=str(target)), strict=False)
|
||||
missing, unexpected = list(missing), list(unexpected)
|
||||
stray = [key for key in missing + unexpected if not key.startswith(TRAINING_ONLY_PREFIX)]
|
||||
if stray:
|
||||
raise ValueError(f"AuK VAE weights at {path} do not match this module: {sorted(stray)[:8]}")
|
||||
logger.info(
|
||||
"Loaded AuK VAE from %s (skipped %d training-only %s* tensors)", path, len(unexpected), TRAINING_ONLY_PREFIX
|
||||
)
|
||||
if fold_weight_norm:
|
||||
self.remove_weight_norm()
|
||||
return missing, unexpected
|
||||
|
||||
def remove_weight_norm(self) -> None:
|
||||
"""Fold ``weight_g``/``weight_v`` into plain weights, on whichever device the module is on.
|
||||
|
||||
Idempotent, and meant to run once the checkpoint is in. See :meth:`load_weights` for why the
|
||||
device this runs on is a numeric decision rather than a convenience.
|
||||
"""
|
||||
if self._norm_folded:
|
||||
return
|
||||
folded = 0
|
||||
for module in self.modules():
|
||||
if hasattr(module, "weight_g"):
|
||||
_fold_weight_norm(module)
|
||||
folded += 1
|
||||
self._norm_folded = True
|
||||
logger.debug("Folded weight norm on %d AuK VAE convolutions", folded)
|
||||
|
||||
def encode(
|
||||
self, wav: torch.Tensor, *, sample: bool = False, generator: torch.Generator | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Encode a mono ``[1, T]`` waveform at :attr:`sample_rate` into ``[1, T / hop_size, latent_dim]``.
|
||||
|
||||
``sample=False`` takes the posterior mean, which is what a reproducible pipeline wants;
|
||||
``sample=True`` draws ``mean + randn * exp(log_std)`` from ``generator``. Latents come back
|
||||
normalized by the checkpoint's global statistics.
|
||||
"""
|
||||
if wav.dim() != 2 or wav.size(0) != 1:
|
||||
raise ValueError(f"encode expects a mono [1, T] waveform, got {tuple(wav.shape)}")
|
||||
# Statistics stay in fp32 even under an ambient autocast, as in the reference.
|
||||
with torch.autocast(device_type=wav.device.type, enabled=False):
|
||||
mean, log_std = self.audio_encoder(wav.unsqueeze(1)).chunk(2, dim=1)
|
||||
if sample:
|
||||
latents = mean + self._noise_like(mean, generator) * torch.exp(log_std)
|
||||
else:
|
||||
latents = mean
|
||||
latents = latents.transpose(1, 2).float()
|
||||
return (latents - self.global_mean.float()) / torch.sqrt(self.global_log_std.float())
|
||||
|
||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
"""Decode normalized ``[B, Np, latent_dim]`` latents into a ``[B, Np * hop_size]`` waveform in [-1, 1]."""
|
||||
if latents.dim() != 3 or latents.size(-1) != self.latent_dim:
|
||||
raise ValueError(f"decode expects [B, Np, {self.latent_dim}] latents, got {tuple(latents.shape)}")
|
||||
x = latents.float() * torch.sqrt(self.global_log_std.float()) + self.global_mean.float()
|
||||
x = self.conv_pre(x.permute(0, 2, 1))
|
||||
for i in range(self.num_upsamples):
|
||||
for up in self.ups[i]:
|
||||
x = up(x)
|
||||
first = i * self.num_kernels
|
||||
residual = self.resblocks[first](x)
|
||||
for j in range(1, self.num_kernels):
|
||||
residual = residual + self.resblocks[first + j](x)
|
||||
x = residual / self.num_kernels
|
||||
x = self.conv_post(self.activation_post(x))
|
||||
return torch.clamp(x, min=-1.0, max=1.0).squeeze(1)
|
||||
|
||||
@staticmethod
|
||||
def _noise_like(reference: torch.Tensor, generator: torch.Generator | None) -> torch.Tensor:
|
||||
"""Draw standard normal noise shaped like ``reference``; ``generator`` decides which RNG stream."""
|
||||
device = reference.device if generator is None else generator.device
|
||||
noise = torch.randn(reference.shape, generator=generator, dtype=reference.dtype, device=device)
|
||||
return noise.to(reference.device)
|
||||
@@ -0,0 +1,486 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""AuK audio-editing pipeline for vLLM-Omni.
|
||||
|
||||
Turns a layer-fused text condition (produced by the encoder stage) plus an
|
||||
optional source clip into a 24 kHz waveform: the source clip is encoded to VAE
|
||||
latents, a rectified-flow DiT integrates the target latents conditioned on
|
||||
both, and the BigVGAN-flow decoder renders the waveform.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchaudio
|
||||
from safetensors import safe_open
|
||||
from torch import nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
|
||||
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
|
||||
from vllm_omni.diffusion.models.interface import (
|
||||
SupportAudioInput,
|
||||
SupportAudioOutput,
|
||||
SupportsComponentDiscovery,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.model_extras.auk import resolve_gen_frames
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Longest ref + target latent sequence the released checkpoints accept.
|
||||
MAX_LATENT_FRAMES = 65536
|
||||
|
||||
# Weight-file layout: the DiT lives under this prefix; the layer-fusion
|
||||
# parameters sit next to it but belong to the encoder stage.
|
||||
_DIT_PREFIX = "transformer."
|
||||
_FUSION_KEYS = frozenset({"layer_weights", "layer_scale"})
|
||||
|
||||
# The distilled variant only reproduces its training recipe at these settings.
|
||||
_FLASH_NFE = 4
|
||||
_FLASH_CFG = 0.0
|
||||
# Decorrelates the VAE posterior draw from the initial latent for the same request seed.
|
||||
_VAE_SEED_OFFSET = 0x5EED_0A0C
|
||||
|
||||
|
||||
def get_auk_post_process_func(od_config: OmniDiffusionConfig):
|
||||
"""Create the post-processing function for AuK audio output.
|
||||
|
||||
Tensor output types pass through; anything else becomes a numpy waveform.
|
||||
The sample rate is not attached here: the output formatter reads
|
||||
``AuKPipeline.audio_sample_rate`` for audio-output pipelines.
|
||||
"""
|
||||
|
||||
del od_config # The conversion does not depend on the config.
|
||||
|
||||
def post_process_func(
|
||||
audio: torch.Tensor,
|
||||
output_type: str = "np",
|
||||
):
|
||||
if output_type in ("pt", "latent"):
|
||||
return audio
|
||||
return audio.cpu().float().numpy()
|
||||
|
||||
return post_process_func
|
||||
|
||||
|
||||
def _prompt_mapping(prompt: Any) -> dict[str, Any]:
|
||||
"""Return the prompt dict, or an empty mapping for a bare text prompt."""
|
||||
|
||||
if isinstance(prompt, dict):
|
||||
return prompt
|
||||
return {}
|
||||
|
||||
|
||||
def _unwrap_single(value: Any) -> Any:
|
||||
"""Unwrap the one-element lists the serving layer sometimes produces."""
|
||||
|
||||
if isinstance(value, list) and len(value) == 1:
|
||||
return value[0]
|
||||
return value
|
||||
|
||||
|
||||
def _split_audio(audio: Any, default_sample_rate: int) -> tuple[Any, int]:
|
||||
"""Split a multimodal audio item into raw samples and a sample rate."""
|
||||
|
||||
if isinstance(audio, (tuple, list)) and len(audio) == 2 and isinstance(audio[1], (int, float)):
|
||||
return audio[0], int(audio[1])
|
||||
return audio, default_sample_rate
|
||||
|
||||
|
||||
class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComponentDiscovery):
|
||||
"""Instruction-driven audio generation and editing with AuK.
|
||||
|
||||
One request per forward: the rectified-flow ODE runs over the whole target
|
||||
span and the batch dimension carries the CFG branches, so there is nothing
|
||||
to share between requests yet.
|
||||
|
||||
Args:
|
||||
od_config: OmniDiffusion configuration. ``od_config.model`` must be an
|
||||
assembled AuK directory (``config.json``, ``auk.safetensors``,
|
||||
``vae.safetensors``), as produced by
|
||||
``tools/prepare_auk_checkpoint.py``.
|
||||
prefix: Unused; kept for the pipeline construction contract.
|
||||
"""
|
||||
|
||||
supports_request_batch = False
|
||||
|
||||
# Picked up by ``supports_audio_output`` in the diffusion engine so the
|
||||
# default stage metadata reports ``final_output_type="audio"`` and the
|
||||
# ``multimodal_output`` payload includes the sample rate.
|
||||
support_audio_output: ClassVar[bool] = True
|
||||
support_audio_input: ClassVar[bool] = True
|
||||
audio_sample_rate: ClassVar[int] = 24000
|
||||
|
||||
# The DiT cannot run without an encoder-stage text condition, which the
|
||||
# engine's synthetic warmup request has no way to produce.
|
||||
dummy_run_num_frames: ClassVar[int] = 0
|
||||
|
||||
_dit_modules: ClassVar[list[str]] = ["dit"]
|
||||
_encoder_modules: ClassVar[list[str]] = []
|
||||
_vae_modules: ClassVar[list[str]] = ["vae"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
od_config: OmniDiffusionConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
del prefix # Weights are not namespaced: one checkpoint, one pipeline.
|
||||
self.od_config = od_config
|
||||
self.device = get_local_device()
|
||||
self.dtype = getattr(od_config, "dtype", None) or torch.bfloat16
|
||||
|
||||
model_dir = od_config.model
|
||||
if not model_dir or not os.path.isdir(model_dir):
|
||||
raise ValueError(
|
||||
f"AuK needs an assembled local checkpoint directory, got {model_dir!r}. "
|
||||
"Build one with tools/prepare_auk_checkpoint.py."
|
||||
)
|
||||
config = _read_config(model_dir)
|
||||
|
||||
vae_config = dict(config["vae"])
|
||||
self.latent_dim = int(vae_config["latent_dim"])
|
||||
self.hop_size = int(vae_config["downsample_rate"])
|
||||
self.sample_rate = int(vae_config["target_sample_rate"])
|
||||
|
||||
self.variant = str(config.get("variant", "base"))
|
||||
self.is_flash = self.variant == "flash"
|
||||
self.flash_t_grid = [float(t) for t in config.get("flash_t_grid") or ()]
|
||||
if self.is_flash and len(self.flash_t_grid) != _FLASH_NFE + 1:
|
||||
raise ValueError(f"The flash variant needs a {_FLASH_NFE + 1}-point flash_t_grid, got {self.flash_t_grid}.")
|
||||
defaults = dict(config.get("defaults") or {})
|
||||
self.default_nfe = int(defaults.get("nfe", 32))
|
||||
self.default_cfg = float(defaults.get("cfg", 2.0))
|
||||
self.default_sway = defaults.get("sway", -1.0)
|
||||
self._flash_lock_logged = False
|
||||
|
||||
# The codec is small and numerically sensitive: it stays in fp32. It
|
||||
# must sit on the inference device BEFORE load_weights folds weight
|
||||
# norm: a CPU fold lands one ULP off the accelerator's and the encoder
|
||||
# amplifies that into a 1e-2 latent drift.
|
||||
self.vae = AuKVAE.from_config(vae_config["model_init_kwargs"]).to(device=self.device, dtype=torch.float32)
|
||||
self.vae.load_weights(os.path.join(model_dir, "vae.safetensors"))
|
||||
self.vae = self.vae.eval()
|
||||
self.vae.requires_grad_(False)
|
||||
self._check_vae_geometry()
|
||||
|
||||
# The transformer owns every key the checkpoint tool writes into `dit`,
|
||||
# `attn_mask_enabled` included, so the section passes straight through.
|
||||
self.dit = AuKTransformer(latent_dim=self.latent_dim, **config["dit"])
|
||||
self.dit = self.dit.to(dtype=self.dtype)
|
||||
self.dit.load_state_dict(_read_dit_weights(model_dir, self.dtype), strict=True)
|
||||
self.dit = self.dit.to(device=self.device).eval()
|
||||
self.dit.requires_grad_(False)
|
||||
|
||||
logger.info(
|
||||
"AuK pipeline ready: variant=%s dtype=%s latent_dim=%d hop=%d sample_rate=%d",
|
||||
self.variant,
|
||||
self.dtype,
|
||||
self.latent_dim,
|
||||
self.hop_size,
|
||||
self.sample_rate,
|
||||
)
|
||||
|
||||
# The assembled checkpoint is not a diffusers layout: __init__ reads
|
||||
# auk.safetensors and vae.safetensors directly, so the loader has no
|
||||
# component sources to stream and load_weights only reports what is loaded.
|
||||
weights_sources: ClassVar[tuple] = ()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stray = [name for name, _ in weights]
|
||||
if stray:
|
||||
logger.warning("AuKPipeline ignores %d loader-provided weights (e.g. %s)", len(stray), stray[:3])
|
||||
return {name for name, _ in self.named_parameters()}
|
||||
|
||||
def _check_vae_geometry(self) -> None:
|
||||
"""Fail fast when config.json and the VAE disagree on latent geometry."""
|
||||
|
||||
declared = {
|
||||
"latent_dim": (self.latent_dim, int(self.vae.latent_dim)),
|
||||
"downsample_rate": (self.hop_size, int(self.vae.hop_size)),
|
||||
"target_sample_rate": (self.sample_rate, int(self.vae.sample_rate)),
|
||||
}
|
||||
bad = {name: pair for name, pair in declared.items() if pair[0] != pair[1]}
|
||||
if bad:
|
||||
raise ValueError(f"config.json and the AuK VAE disagree (config, vae): {bad}")
|
||||
if self.sample_rate != self.audio_sample_rate:
|
||||
raise ValueError(
|
||||
f"AuK advertises {self.audio_sample_rate} Hz output but the checkpoint is {self.sample_rate} Hz."
|
||||
)
|
||||
|
||||
def _resolve_generator(self, sampling_params: Any) -> torch.Generator | None:
|
||||
"""Per-request generator; the process-global RNG is never seeded."""
|
||||
|
||||
generator = sampling_params.generator
|
||||
if isinstance(generator, list):
|
||||
if len(generator) > 1:
|
||||
logger.warning(
|
||||
"AuKPipeline runs one request per forward; using the first of %d generators", len(generator)
|
||||
)
|
||||
generator = generator[0] if generator else None
|
||||
if generator is None and sampling_params.seed is not None:
|
||||
generator = torch.Generator(device=self.device).manual_seed(int(sampling_params.seed))
|
||||
return generator
|
||||
|
||||
def _prepare_waveform(self, audio: Any) -> torch.Tensor:
|
||||
"""Return mono float32 samples ``[1, T]`` at the VAE sample rate.
|
||||
|
||||
``T`` is truncated to a whole number of latent frames. The encoder is
|
||||
causal, so dropping a trailing partial frame leaves every emitted frame
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
data, sample_rate = _split_audio(audio, self.sample_rate)
|
||||
if isinstance(data, np.ndarray):
|
||||
wav = torch.from_numpy(np.ascontiguousarray(data))
|
||||
else:
|
||||
wav = torch.as_tensor(data)
|
||||
wav = wav.detach().to(device="cpu", dtype=torch.float32)
|
||||
|
||||
if wav.ndim == 1:
|
||||
wav = wav.unsqueeze(0)
|
||||
elif wav.ndim == 2:
|
||||
# Accept both [channels, samples] and soundfile's [samples, channels];
|
||||
# the channel axis is the short one (at most 8 wide).
|
||||
if wav.shape[0] > wav.shape[1] and wav.shape[1] <= 8:
|
||||
wav = wav.transpose(0, 1)
|
||||
if wav.shape[0] > 1:
|
||||
wav = wav.mean(dim=0, keepdim=True)
|
||||
else:
|
||||
raise ValueError(f"AuK audio input must be [samples] or [channels, samples], got {tuple(wav.shape)}.")
|
||||
|
||||
if wav.shape[-1] == 0:
|
||||
raise ValueError("AuK audio input is empty.")
|
||||
if not torch.isfinite(wav).all():
|
||||
raise ValueError("AuK audio input contains NaN or Inf.")
|
||||
|
||||
if sample_rate != self.sample_rate:
|
||||
wav = torchaudio.functional.resample(wav, sample_rate, self.sample_rate)
|
||||
|
||||
frames = wav.shape[-1] // self.hop_size
|
||||
if frames < 1:
|
||||
raise ValueError(
|
||||
f"AuK audio input is shorter than one latent frame ({self.hop_size} samples at {self.sample_rate} Hz)."
|
||||
)
|
||||
return wav[..., : frames * self.hop_size]
|
||||
|
||||
def _encode_source(
|
||||
self,
|
||||
audio: Any,
|
||||
*,
|
||||
sample: bool,
|
||||
generator: torch.Generator | None,
|
||||
) -> torch.Tensor:
|
||||
"""Encode the source clip to normalized latents ``[1, np, latent_dim]``."""
|
||||
|
||||
if audio is None:
|
||||
return torch.zeros(1, 0, self.latent_dim, device=self.device, dtype=torch.float32)
|
||||
wav = self._prepare_waveform(_unwrap_single(audio)).to(self.device)
|
||||
return self.vae.encode(wav, sample=sample, generator=generator)
|
||||
|
||||
def _target_frames(self, gen_seconds: Any, ref_frames: int) -> int:
|
||||
"""Target latent length, capped by the remaining context."""
|
||||
|
||||
frames = resolve_gen_frames(
|
||||
None if gen_seconds is None else float(gen_seconds),
|
||||
ref_frames,
|
||||
sample_rate=self.sample_rate,
|
||||
hop=self.hop_size,
|
||||
)
|
||||
budget = MAX_LATENT_FRAMES - ref_frames
|
||||
if budget < 1:
|
||||
raise ValueError(
|
||||
f"The source clip already fills the {MAX_LATENT_FRAMES}-frame context ({ref_frames} frames)."
|
||||
)
|
||||
if frames > budget:
|
||||
logger.warning("AuK target of %d frames exceeds the remaining context; clamping to %d.", frames, budget)
|
||||
frames = budget
|
||||
return frames
|
||||
|
||||
def _resolve_schedule(
|
||||
self,
|
||||
sampling_params: Any,
|
||||
knobs: dict[str, Any],
|
||||
) -> tuple[int, float, float | None, list[float] | None]:
|
||||
"""Resolve (nfe, cfg, sway, t_grid), locking the distilled recipe on Flash.
|
||||
|
||||
A knob present but ``None`` counts as unspecified: the stage input
|
||||
processor fills every key it knows about, whether the caller set it or
|
||||
not.
|
||||
"""
|
||||
|
||||
steps = sampling_params.num_inference_steps
|
||||
nfe = self.default_nfe if steps is None else int(steps)
|
||||
if nfe < 1:
|
||||
raise ValueError(f"num_inference_steps must be >= 1 for AuK; got {steps}")
|
||||
cfg = float(sampling_params.guidance_scale) if sampling_params.guidance_scale_provided else self.default_cfg
|
||||
sway = knobs.get("sway")
|
||||
sway = self.default_sway if sway is None else float(sway)
|
||||
raw_grid = knobs.get("t_grid")
|
||||
t_grid = [float(t) for t in raw_grid] if raw_grid else None
|
||||
|
||||
if not self.is_flash:
|
||||
return nfe, cfg, sway, t_grid
|
||||
|
||||
locked = (_FLASH_NFE, _FLASH_CFG, None, self.flash_t_grid)
|
||||
if (nfe, cfg, sway, t_grid) != locked and not self._flash_lock_logged:
|
||||
self._flash_lock_logged = True
|
||||
logger.warning(
|
||||
"AuK-Flash ignores per-request sampling knobs (asked nfe=%s cfg=%s sway=%s t_grid=%s); "
|
||||
"using the distilled recipe nfe=%d cfg=%s with the checkpoint time grid.",
|
||||
nfe,
|
||||
cfg,
|
||||
sway,
|
||||
t_grid,
|
||||
_FLASH_NFE,
|
||||
_FLASH_CFG,
|
||||
)
|
||||
return locked
|
||||
|
||||
def _dit_autocast(self):
|
||||
"""Match the reference, which runs the DiT under bf16 autocast."""
|
||||
|
||||
return torch.autocast(
|
||||
device_type=self.device.type,
|
||||
dtype=self.dtype,
|
||||
enabled=self.dtype in (torch.bfloat16, torch.float16),
|
||||
)
|
||||
|
||||
def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]:
|
||||
"""Generate one waveform.
|
||||
|
||||
Args:
|
||||
req: Request batch holding exactly one request. The prompt carries
|
||||
``prompt_embeds`` (the fused text condition ``[nt, 2048]``), an
|
||||
optional ``multi_modal_data["audio"]`` source clip, and
|
||||
``additional_information["auk"]`` with ``gen_seconds``,
|
||||
``sway``, ``t_grid`` and ``vae_sample``. ``num_inference_steps``,
|
||||
``guidance_scale`` and ``seed``/``generator`` come from the
|
||||
sampling params.
|
||||
|
||||
Returns:
|
||||
One ``DiffusionOutput`` whose ``output`` is a float32 mono waveform
|
||||
``[T]`` at 24 kHz, or the normalized target latents
|
||||
``[1, gen_frames, latent_dim]`` when ``output_type`` is ``latent``.
|
||||
"""
|
||||
|
||||
assert req.num_reqs == 1, f"AuKPipeline runs one request per forward, got {req.num_reqs}."
|
||||
prompt = _prompt_mapping(req.prompts[0])
|
||||
sampling_params = req.sampling_params
|
||||
knobs = dict((prompt.get("additional_information") or {}).get("auk") or {})
|
||||
|
||||
text = _unwrap_single(prompt.get("prompt_embeds"))
|
||||
if text is None:
|
||||
raise ValueError("AuK stage 1 needs `prompt_embeds`, the fused text condition from the encoder stage.")
|
||||
text = torch.as_tensor(text).to(device=self.device, dtype=self.dtype)
|
||||
if text.ndim == 3 and text.shape[0] == 1:
|
||||
text = text[0]
|
||||
if text.ndim != 2:
|
||||
raise ValueError(f"AuK `prompt_embeds` must be [nt, text_hidden_dim], got {tuple(text.shape)}.")
|
||||
text = text.unsqueeze(0)
|
||||
c_mask = torch.ones(text.shape[:2], dtype=torch.bool, device=self.device)
|
||||
|
||||
generator = self._resolve_generator(sampling_params)
|
||||
audio = (prompt.get("multi_modal_data") or {}).get("audio")
|
||||
output_type = sampling_params.output_type or "np"
|
||||
|
||||
with torch.inference_mode():
|
||||
# The VAE posterior draw gets its own generator, seeded with a
|
||||
# fixed offset from the request seed: the initial latent below then
|
||||
# still equals the reference's fresh manual_seed draw, and the two
|
||||
# noise streams are not copies of each other.
|
||||
vae_generator = None
|
||||
if knobs.get("vae_sample") and generator is not None:
|
||||
vae_seed = (int(generator.initial_seed()) + _VAE_SEED_OFFSET) % (2**63)
|
||||
vae_generator = torch.Generator(device=self.device).manual_seed(vae_seed)
|
||||
ref = self._encode_source(audio, sample=bool(knobs.get("vae_sample")), generator=vae_generator)
|
||||
ref_frames = ref.shape[1]
|
||||
ref_mask = torch.ones(ref.shape[:2], dtype=torch.bool, device=self.device)
|
||||
gen_frames = self._target_frames(knobs.get("gen_seconds"), ref_frames)
|
||||
nfe, cfg, sway, t_grid = self._resolve_schedule(sampling_params, knobs)
|
||||
|
||||
with self._dit_autocast():
|
||||
latents = sample_latents(
|
||||
self.dit,
|
||||
text=text,
|
||||
c_mask=c_mask,
|
||||
ref=ref,
|
||||
ref_mask=ref_mask,
|
||||
gen_frames=gen_frames,
|
||||
nfe=nfe,
|
||||
cfg_strength=cfg,
|
||||
sway_sampling_coef=sway,
|
||||
t_grid=t_grid,
|
||||
seed=None,
|
||||
latent_dim=self.latent_dim,
|
||||
device=self.device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
)
|
||||
|
||||
latents = latents.float()
|
||||
if not torch.isfinite(latents).all():
|
||||
raise RuntimeError("AuK generated latents contain NaN or Inf.")
|
||||
if output_type == "latent":
|
||||
return [DiffusionOutput(output=latents.detach().cpu())]
|
||||
|
||||
wav = self.vae.decode(latents)
|
||||
|
||||
# One mono waveform per request; the formatter expects [T].
|
||||
wav = wav.detach().to(device="cpu", dtype=torch.float32).reshape(-1)
|
||||
if not torch.isfinite(wav).all():
|
||||
raise RuntimeError("AuK generated audio contains NaN or Inf.")
|
||||
return [DiffusionOutput(output=wav)]
|
||||
|
||||
|
||||
def _read_config(model_dir: str) -> dict[str, Any]:
|
||||
"""Read the assembled AuK ``config.json``."""
|
||||
|
||||
path = os.path.join(model_dir, "config.json")
|
||||
try:
|
||||
with open(path) as handle:
|
||||
config = json.load(handle)
|
||||
except FileNotFoundError as exc:
|
||||
raise ValueError(f"AuK checkpoint is missing config.json: {path}") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.error("AuK config.json is not valid JSON: %s", path)
|
||||
raise ValueError(f"AuK config.json is not valid JSON: {path}") from exc
|
||||
|
||||
missing = [key for key in ("dit", "vae") if key not in config]
|
||||
if missing:
|
||||
raise ValueError(f"AuK config.json is missing required sections {missing}: {path}")
|
||||
return config
|
||||
|
||||
|
||||
def _read_dit_weights(model_dir: str, dtype: torch.dtype) -> dict[str, torch.Tensor]:
|
||||
"""Read the DiT state dict from ``auk.safetensors``, cast to ``dtype``.
|
||||
|
||||
Tensors are cast one at a time so the fp32 checkpoint is never fully
|
||||
resident alongside the model.
|
||||
"""
|
||||
|
||||
path = os.path.join(model_dir, "auk.safetensors")
|
||||
if not os.path.isfile(path):
|
||||
raise ValueError(f"AuK checkpoint is missing auk.safetensors: {path}")
|
||||
|
||||
with safe_open(path, framework="pt", device="cpu") as checkpoint:
|
||||
keys = list(checkpoint.keys())
|
||||
state = dit_state_dict(((key, checkpoint.get_tensor(key).to(dtype)) for key in keys), _DIT_PREFIX)
|
||||
|
||||
if not state:
|
||||
raise ValueError(f"AuK checkpoint has no '{_DIT_PREFIX}*' weights: {path}")
|
||||
unexpected = sorted(key for key in keys if key not in _FUSION_KEYS and not key.startswith(_DIT_PREFIX))
|
||||
if unexpected:
|
||||
logger.warning("Ignoring %d unrecognized keys in %s, e.g. %s", len(unexpected), path, unexpected[:5])
|
||||
return state
|
||||
Reference in new issue
Block a user