[Model] Add Tencent AuK speech generation and editing (encoder + diffusion pipeline) (#7385)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Yueqian LinandSy03 authored and GitHub committed 2026-09-11 20:09:58 +00:00
1 parent d3493384cf
commit c111f3a2d6
28 files changed
+4986

No files matched your search

@@ -0,0 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""AuK diffusion-stage components: the flow transformer, the codec, the pipeline."""
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
from vllm_omni.diffusion.models.auk.pipeline_auk import AuKPipeline, get_auk_post_process_func
__all__ = [
"AuKPipeline",
"AuKTransformer",
"AuKVAE",
"dit_state_dict",
"get_auk_post_process_func",
"sample_latents",
]
@@ -0,0 +1,677 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The AuK audio DiT as a self-contained inference module.
Two phases. ``num_layers`` double-stream blocks attend jointly over text and
audio while keeping each stream's residual separate, then
``num_single_layers`` single-stream blocks run over the concatenated
``[text | ref audio | target audio]`` sequence. Only the target-audio slice is
projected back to latent space, so the module predicts a flow-matching
velocity for the target frames and :func:`sample_latents` integrates it.
Parameter names and shapes match the AuK reference backbone, so a released
checkpoint loads with ``load_state_dict(strict=True)`` once the
``transformer.`` prefix is stripped (see :func:`dit_state_dict`).
Deliberate differences from the reference, all inference-only:
* ``x_transformers`` and ``torchdiffeq`` are gone. Rotary embeddings are
computed here (interleaved GPT-J pairs, no xpos scaling) and the ODE is an
explicit Euler loop.
* Dropout, activation checkpointing, the ``flash_attn`` backend switch and the
zero-init of the modulation layers are training concerns and are dropped.
* The timestep sinusoid is cast to the time MLP's weight dtype rather than to
the timestep's own dtype, so an fp32 timestep works against half-precision
weights outside autocast.
* ``attn_mask_enabled`` defaults to ``True``, the value the released config
sets, rather than the reference's ``False``.
"""
import math
from collections.abc import Iterable, Mapping
import torch
import torch.nn.functional as F
from torch import nn
__all__ = ["AuKTransformer", "dit_state_dict", "sample_latents"]
def _sdpa(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor:
"""The single attention call site, so the backend can be swapped in one place.
``q``/``k``/``v`` are ``[B, H, N, D]``. ``mask`` is a boolean key-padding
mask ``[B, K]`` in which ``True`` marks a valid key; it is broadcast to
``[B, H, Q, K]`` only when given.
"""
attn_mask = None
if mask is not None:
attn_mask = mask[:, None, None, :].expand(q.shape[0], q.shape[1], q.shape[-2], k.shape[-2])
return F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask, dropout_p=0.0, is_causal=False)
def _rotate_half(x: torch.Tensor) -> torch.Tensor:
"""Rotate interleaved GPT-J pairs: ``(x1, x2) -> (-x2, x1)``."""
pairs = x.reshape(*x.shape[:-1], -1, 2)
x1, x2 = pairs.unbind(dim=-1)
return torch.stack((-x2, x1), dim=-1).flatten(-2)
def _apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
"""Apply rotary frequencies ``[1, 1, N, D]`` to ``x`` ``[B, H, N, D]`` in fp32."""
out_dtype = x.dtype
x32 = x.float()
out = x32 * freqs.cos() + _rotate_half(x32) * freqs.sin()
return out.to(out_dtype)
class Rotary(nn.Module):
"""Rotary position frequencies for one stream, evaluated in fp32.
The frequencies live in a buffer so that a released checkpoint's own copy
of them loads with ``strict=True``. They are recomputed whenever the
module has been cast below fp32: rounding either the frequencies or the
integer positions to bf16 wrecks the phase at long positions.
"""
def __init__(self, dim: int, base: float = 10000.0) -> None:
super().__init__()
if dim % 2 != 0:
raise ValueError(f"rotary dim must be even, got {dim}")
self.dim = dim
self.base = base
self.register_buffer("inv_freq", self._frequencies(torch.device("cpu")))
def _frequencies(self, device: torch.device) -> torch.Tensor:
exponents = torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim
return 1.0 / (self.base**exponents)
def forward(self, seq_len: int) -> torch.Tensor:
"""Return frequencies ``[1, 1, seq_len, dim]`` for positions ``0..seq_len-1``."""
inv_freq = self.inv_freq
if inv_freq.dtype != torch.float32:
inv_freq = self._frequencies(inv_freq.device)
pos = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32)
freqs = torch.outer(pos, inv_freq)
return torch.stack((freqs, freqs), dim=-1).flatten(-2)[None, None]
class TimeEmbedding(nn.Module):
"""Sinusoidal timestep features followed by a two-layer MLP."""
def __init__(self, dim: int, freq_embed_dim: int = 256) -> None:
super().__init__()
self.freq_embed_dim = freq_embed_dim
self.time_mlp = nn.Sequential(nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim))
def forward(self, timestep: torch.Tensor, scale: float = 1000.0) -> torch.Tensor:
half = self.freq_embed_dim // 2
decay = math.log(10000) / (half - 1)
freqs = torch.exp(torch.arange(half, device=timestep.device, dtype=torch.float32) * -decay)
angles = scale * timestep.unsqueeze(1) * freqs.unsqueeze(0)
hidden = torch.cat((angles.sin(), angles.cos()), dim=-1)
return self.time_mlp(hidden.to(self.time_mlp[0].weight.dtype))
class ConvPosEmbedding(nn.Module):
"""Depthwise-grouped conv stack that adds local position information."""
def __init__(self, dim: int, kernel_size: int = 31, groups: int = 16) -> None:
super().__init__()
if kernel_size % 2 == 0:
raise ValueError(f"kernel_size must be odd, got {kernel_size}")
padding = kernel_size // 2
self.conv1d = nn.Sequential(
nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=padding),
nn.Mish(),
nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=padding),
nn.Mish(),
)
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
keep = None if mask is None else mask.unsqueeze(1)
x = x.transpose(1, 2)
if keep is not None:
x = x.masked_fill(~keep, 0.0)
for layer in self.conv1d:
x = layer(x)
# Re-zero padding after each convolution so it cannot bleed into
# valid frames through the kernel window.
if keep is not None and isinstance(layer, nn.Conv1d):
x = x.masked_fill(~keep, 0.0)
return x.transpose(1, 2)
class AudioEmbedding(nn.Module):
"""Project audio latents to model width and add conv position information."""
def __init__(self, latent_dim: int, dim: int) -> None:
super().__init__()
self.linear = nn.Linear(latent_dim, dim)
self.conv_pos_embed = ConvPosEmbedding(dim)
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
x = self.linear(x)
return self.conv_pos_embed(x, mask=mask) + x
class AdaLayerNorm(nn.Module):
"""Timestep-conditioned modulation for a block: six chunks from one projection."""
def __init__(self, dim: int) -> None:
super().__init__()
self.silu = nn.SiLU()
self.linear = nn.Linear(dim, dim * 6)
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
def forward(
self, x: torch.Tensor, emb: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.linear(self.silu(emb)).chunk(6, dim=1)
x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
return x, gate_msa, shift_mlp, scale_mlp, gate_mlp
class AdaLayerNormFinal(nn.Module):
"""Timestep-conditioned modulation before the output projection."""
def __init__(self, dim: int) -> None:
super().__init__()
self.silu = nn.SiLU()
self.linear = nn.Linear(dim, dim * 2)
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor:
scale, shift = self.linear(self.silu(emb)).chunk(2, dim=1)
return self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
class FeedForward(nn.Module):
"""SwiGLU feed-forward with the gate projection fused into ``linear_in``."""
def __init__(self, dim: int, mult: float = 4.0) -> None:
super().__init__()
inner_dim = int(dim * mult)
self.linear_in = nn.Linear(dim, inner_dim * 2, bias=False)
self.linear_out = nn.Linear(inner_dim, dim, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate, up = self.linear_in(x).chunk(2, dim=-1)
return self.linear_out(F.silu(gate) * up)
class Attention(nn.Module):
"""Self-attention with per-head QK RMSNorm and rotary positions.
``to_out`` stays a ``ModuleList`` because the released checkpoints name the
output projection ``to_out.0``; the reference's trailing dropout carries no
parameters and is dropped.
"""
def __init__(self, dim: int, heads: int, dim_head: int, attn_mask_enabled: bool = True) -> None:
super().__init__()
self.heads = heads
self.dim_head = dim_head
self.attn_mask_enabled = attn_mask_enabled
inner_dim = heads * dim_head
self.to_qkv = nn.Linear(dim, 3 * inner_dim)
self.q_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
self.k_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
self.to_out = nn.ModuleList([nn.Linear(inner_dim, dim)])
def _merge_heads(self, x: torch.Tensor) -> torch.Tensor:
return x.transpose(1, 2).reshape(x.shape[0], -1, self.heads * self.dim_head)
def _qkv(
self,
packed: torch.Tensor,
q_norm: nn.Module,
k_norm: nn.Module,
rope: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Split a packed QKV projection into heads, RMSNorm Q/K, then rotate."""
q, k, v = (t.view(t.shape[0], -1, self.heads, self.dim_head).transpose(1, 2) for t in packed.chunk(3, dim=-1))
q = q_norm(q)
k = k_norm(k)
if rope is not None:
q = _apply_rope(q, rope)
k = _apply_rope(k, rope)
return q, k, v
def _project(self, x: torch.Tensor, rope: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return self._qkv(self.to_qkv(x), self.q_norm, self.k_norm, rope)
def forward(
self, x: torch.Tensor, mask: torch.Tensor | None = None, rope: torch.Tensor | None = None
) -> torch.Tensor:
q, k, v = self._project(x, rope)
attn_mask = mask if self.attn_mask_enabled else None
out = self._merge_heads(_sdpa(q, k, v, attn_mask).to(q.dtype))
out = self.to_out[0](out)
if mask is not None:
out = out.masked_fill(~mask.unsqueeze(-1), 0.0)
return out
class JointAttention(Attention):
"""Attention over the audio and text streams jointly, with separate projections."""
def __init__(self, dim: int, heads: int, dim_head: int, context_dim: int, attn_mask_enabled: bool = True) -> None:
super().__init__(dim, heads, dim_head, attn_mask_enabled)
inner_dim = heads * dim_head
self.to_qkv_c = nn.Linear(context_dim, 3 * inner_dim)
self.c_q_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
self.c_k_norm = nn.RMSNorm(dim_head, elementwise_affine=True)
self.to_out_c = nn.Linear(inner_dim, context_dim)
def _project_context(self, c: torch.Tensor, c_rope: torch.Tensor | None) -> tuple[torch.Tensor, ...]:
return self._qkv(self.to_qkv_c(c), self.c_q_norm, self.c_k_norm, c_rope)
def forward( # type: ignore[override] # joint attention takes the context stream too
self,
x: torch.Tensor,
c: torch.Tensor,
mask: torch.Tensor | None = None,
rope: torch.Tensor | None = None,
c_rope: torch.Tensor | None = None,
c_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
audio_len = x.shape[1]
q, k, v = self._project(x, rope)
c_q, c_k, c_v = self._project_context(c, c_rope)
q = torch.cat([q, c_q], dim=2)
k = torch.cat([k, c_k], dim=2)
v = torch.cat([v, c_v], dim=2)
joint_mask = None
if self.attn_mask_enabled and mask is not None:
if c_mask is not None:
joint_mask = torch.cat([mask, c_mask], dim=1)
else:
joint_mask = F.pad(mask, (0, c.shape[1]), value=True)
out = self._merge_heads(_sdpa(q, k, v, joint_mask).to(q.dtype))
x_out = self.to_out[0](out[:, :audio_len])
c_out = self.to_out_c(out[:, audio_len:])
if mask is not None:
x_out = x_out.masked_fill(~mask.unsqueeze(-1), 0.0)
if c_mask is not None:
c_out = c_out.masked_fill(~c_mask.unsqueeze(-1), 0.0)
return x_out, c_out
class DoubleBlock(nn.Module):
"""MM-DiT block: joint attention, separate modulation and feed-forward per stream."""
def __init__(self, dim: int, heads: int, dim_head: int, ff_mult: float, attn_mask_enabled: bool) -> None:
super().__init__()
self.attn_norm_c = AdaLayerNorm(dim)
self.attn_norm_x = AdaLayerNorm(dim)
self.attn = JointAttention(
dim=dim, heads=heads, dim_head=dim_head, context_dim=dim, attn_mask_enabled=attn_mask_enabled
)
self.ff_norm_c = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff_c = FeedForward(dim=dim, mult=ff_mult)
self.ff_norm_x = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff_x = FeedForward(dim=dim, mult=ff_mult)
def forward(
self,
x: torch.Tensor,
c: torch.Tensor,
t: torch.Tensor,
mask: torch.Tensor | None,
rope: torch.Tensor,
c_rope: torch.Tensor,
c_mask: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t)
norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t)
x_attn, c_attn = self.attn(x=norm_x, c=norm_c, mask=mask, rope=rope, c_rope=c_rope, c_mask=c_mask)
c = c + c_gate_msa[:, None] * c_attn
norm_c = self.ff_norm_c(c) * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None]
c = c + c_gate_mlp[:, None] * self.ff_c(norm_c)
x = x + x_gate_msa[:, None] * x_attn
norm_x = self.ff_norm_x(x) * (1 + x_scale_mlp[:, None]) + x_shift_mlp[:, None]
x = x + x_gate_mlp[:, None] * self.ff_x(norm_x)
return c, x
class SingleBlock(nn.Module):
"""DiT block over the concatenated text and audio sequence."""
def __init__(self, dim: int, heads: int, dim_head: int, ff_mult: float, attn_mask_enabled: bool) -> None:
super().__init__()
self.attn_norm = AdaLayerNorm(dim)
self.attn = Attention(dim=dim, heads=heads, dim_head=dim_head, attn_mask_enabled=attn_mask_enabled)
self.ff_norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff = FeedForward(dim=dim, mult=ff_mult)
def forward(self, x: torch.Tensor, t: torch.Tensor, mask: torch.Tensor | None, rope: torch.Tensor) -> torch.Tensor:
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t)
x = x + gate_msa[:, None] * self.attn(x=norm, mask=mask, rope=rope)
norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
return x + gate_mlp[:, None] * self.ff(norm)
class AuKTransformer(nn.Module):
"""Flow-matching velocity predictor for AuK audio latents.
Args:
dim: Model width.
heads: Attention heads.
dim_head: Per-head width.
ff_mult: Feed-forward expansion factor.
latent_dim: Audio VAE latent channels, the input and output width.
text_hidden_dim: Width of the pre-encoded text hidden states.
num_layers: Double-stream (joint attention) block count.
num_single_layers: Single-stream block count.
attn_mask_enabled: Apply padding masks inside attention. When ``False``
attention runs unmasked, though padded outputs are still zeroed,
which is what the reference does with the flag off.
"""
def __init__(
self,
*,
dim: int,
heads: int,
dim_head: int,
ff_mult: float,
latent_dim: int,
text_hidden_dim: int,
num_layers: int,
num_single_layers: int,
attn_mask_enabled: bool = True,
) -> None:
super().__init__()
self.dim = dim
self.latent_dim = latent_dim
self.text_cond: torch.Tensor | None = None
self.text_uncond: torch.Tensor | None = None
self.time_embed = TimeEmbedding(dim)
self.txt_norm = nn.RMSNorm(dim, elementwise_affine=True)
self.txt_proj = nn.Linear(text_hidden_dim, dim)
self.audio_embed = AudioEmbedding(latent_dim, dim)
self.rotary_embed = Rotary(dim_head)
self.transformer_blocks = nn.ModuleList(
[
DoubleBlock(
dim=dim, heads=heads, dim_head=dim_head, ff_mult=ff_mult, attn_mask_enabled=attn_mask_enabled
)
for _ in range(num_layers)
]
)
self.single_transformer_blocks = nn.ModuleList(
[
SingleBlock(
dim=dim, heads=heads, dim_head=dim_head, ff_mult=ff_mult, attn_mask_enabled=attn_mask_enabled
)
for _ in range(num_single_layers)
]
)
self.norm_out = AdaLayerNormFinal(dim)
self.proj_out = nn.Linear(dim, latent_dim)
def project_text(self, text: torch.Tensor) -> torch.Tensor:
"""Project LLM hidden states ``[B, nt, text_hidden_dim]`` to model width."""
return self.txt_norm(self.txt_proj(text))
def clear_cache(self) -> None:
"""Drop the cached text projection. Call this when the conditioning changes."""
self.text_cond = None
self.text_uncond = None
def _embed_audio(
self,
x: torch.Tensor,
ref: torch.Tensor | None,
drop_audio_cond: bool,
mask: torch.Tensor | None,
ref_mask: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None, int]:
"""Embed the target latents, prepending the reference prompt when present.
Returns the ``[ref | target]`` sequence, its padding mask and the
reference length, which the caller uses to slice the target back out.
"""
if ref is not None and ref.shape[1] == 0:
ref = None
target = self.audio_embed(x, mask=mask)
if ref is None:
return target, mask, 0
if drop_audio_cond:
ref = torch.zeros_like(ref)
prompt = self.audio_embed(ref, mask=ref_mask)
audio = torch.cat([prompt, target], dim=1)
prompt_len = prompt.shape[1]
audio_mask = None
if mask is not None or ref_mask is not None:
if mask is None:
mask = x.new_ones(x.shape[:2], dtype=torch.bool)
if ref_mask is None:
ref_mask = ref.new_ones(ref.shape[:2], dtype=torch.bool)
audio_mask = torch.cat([ref_mask, mask], dim=1)
return audio, audio_mask, prompt_len
def forward(
self,
x: torch.Tensor,
text: torch.Tensor,
time: torch.Tensor,
*,
mask: torch.Tensor | None = None,
c_mask: torch.Tensor | None = None,
ref: torch.Tensor | None = None,
ref_mask: torch.Tensor | None = None,
drop_audio_cond: bool = False,
drop_text: bool = False,
cfg_infer: bool = False,
cache: bool = False,
) -> torch.Tensor:
"""Predict the flow-matching velocity for the target frames of ``x``.
Args:
x: Noised target latents ``[B, n, latent_dim]``.
text: Pre-encoded text hidden states ``[B, nt, text_hidden_dim]``.
time: Flow-matching timestep, a scalar or ``[B]``.
mask: Target padding mask ``[B, n]``, ``True`` where valid.
c_mask: Text padding mask ``[B, nt]``. Inferred from all-zero text
rows when omitted.
ref: Reference prompt latents ``[B, np, latent_dim]``, prepended to
the audio sequence. A zero-length ``ref`` counts as absent.
ref_mask: Reference padding mask ``[B, np]``.
drop_audio_cond: Zero the reference prompt, for the CFG uncond branch.
drop_text: Zero the projected text, for the CFG uncond branch.
cfg_infer: Run the cond and uncond branches as one batch of ``2B``,
cond first. ``drop_audio_cond`` and ``drop_text`` are then
driven per branch and ignored.
cache: Reuse the projected text across calls, for an ODE loop over
fixed conditioning. Only the ``cfg_infer`` path caches, as in
the reference. Call :meth:`clear_cache` when the text changes.
Returns:
Velocity ``[B, n, latent_dim]``, or ``[2B, n, latent_dim]`` under
``cfg_infer``.
"""
batch = x.shape[0]
if time.ndim == 0:
time = time.repeat(batch)
t = self.time_embed(time)
if c_mask is None:
# Padding rows are all-zero in the text encoder's output.
c_mask = text.abs().sum(-1) > 0
if cfg_infer:
if cache and self.text_cond is not None:
c_cond, c_uncond = self.text_cond, self.text_uncond
else:
c_cond = self.project_text(text)
# The uncond branch drops the text, which is exactly a zeroed projection.
c_uncond = torch.zeros_like(c_cond)
if cache:
self.text_cond, self.text_uncond = c_cond, c_uncond
x_cond, mask_cond, prompt_len = self._embed_audio(x, ref, False, mask, ref_mask)
x_uncond, mask_uncond, _ = self._embed_audio(x, ref, True, mask, ref_mask)
x = torch.cat((x_cond, x_uncond), dim=0)
c = torch.cat((c_cond, c_uncond), dim=0)
t = torch.cat((t, t), dim=0)
c_mask = torch.cat((c_mask, c_mask), dim=0)
audio_mask = None
if mask_cond is not None and mask_uncond is not None:
audio_mask = torch.cat((mask_cond, mask_uncond), dim=0)
else:
c = self.project_text(text)
if drop_text:
c = torch.zeros_like(c)
x, audio_mask, prompt_len = self._embed_audio(x, ref, drop_audio_cond, mask, ref_mask)
seq_len = x.shape[1]
text_len = c.shape[1]
rope_audio = self.rotary_embed(seq_len)
rope_text = self.rotary_embed(text_len)
for block in self.transformer_blocks:
c, x = block(x, c, t, mask=audio_mask, rope=rope_audio, c_rope=rope_text, c_mask=c_mask)
x = torch.cat([c, x], dim=1)
rope = self.rotary_embed(text_len + seq_len)
single_mask = None if audio_mask is None else torch.cat([c_mask, audio_mask], dim=1)
for block in self.single_transformer_blocks:
x = block(x, t, mask=single_mask, rope=rope)
x = x[:, text_len + prompt_len :]
return self.proj_out(self.norm_out(x, t))
def dit_state_dict(
checkpoint: Mapping[str, torch.Tensor] | Iterable[tuple[str, torch.Tensor]],
prefix: str = "transformer.",
) -> dict[str, torch.Tensor]:
"""Select the DiT tensors out of a full AuK checkpoint and strip ``prefix``.
Released checkpoints wrap the backbone under ``transformer.`` alongside the
text-encoder layer-fusion parameters, which this module does not own.
"""
items = checkpoint.items() if isinstance(checkpoint, Mapping) else checkpoint
return {name[len(prefix) :]: tensor for name, tensor in items if name.startswith(prefix)}
@torch.no_grad()
def sample_latents(
dit: AuKTransformer,
*,
text: torch.Tensor,
c_mask: torch.Tensor | None,
ref: torch.Tensor,
ref_mask: torch.Tensor | None,
gen_frames: int,
nfe: int = 32,
cfg_strength: float = 1.0,
sway_sampling_coef: float | None = None,
t_grid: list[float] | None = None,
seed: int | None = None,
generator: torch.Generator | None = None,
latent_dim: int | None = None,
device: torch.device | str | None = None,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""Integrate the flow from noise to audio latents with explicit Euler steps.
Single-request only: the batch dimension carries the CFG branches, not
separate prompts, so no target padding mask is needed.
Args:
dit: The velocity model.
text: Pre-encoded text hidden states ``[1, nt, text_hidden_dim]``.
c_mask: Text padding mask ``[1, nt]``.
ref: Reference prompt latents ``[1, np, latent_dim]``.
ref_mask: Reference padding mask ``[1, np]``.
gen_frames: Target latent frames to generate.
nfe: Euler steps, ignored when ``t_grid`` is given.
cfg_strength: Classifier-free guidance weight. Below ``1e-5`` the
uncond branch is skipped entirely.
sway_sampling_coef: Reshapes the uniform time grid towards ``t=0``.
t_grid: Explicit timesteps, overriding ``nfe`` and the sway reshape.
seed: Seeds the global RNG before drawing the noise. Ignored when
``generator`` is given.
generator: Draws the initial noise from its own RNG, leaving the global
one untouched.
latent_dim: Latent channels; defaults to the model's.
device: Device for the noise; defaults to the reference's.
dtype: Dtype for the noise and the Euler accumulator; defaults to the
reference's, which is the reference implementation's rule. Pass
``torch.float32`` explicitly when serving half-precision weights.
Drawing the noise at bf16 instead of fp32 moves a 32-step CFG
trajectory by 0.025 relative MSE, roughly 17 times what bf16
weights themselves cost, because the draw consumes the generator
differently and starts the ODE somewhere else. Keeping it fp32
costs a few tens of KB and no measurable time.
Returns:
The target latents at ``t=1``, ``[1, gen_frames, latent_dim]``.
"""
device = torch.device(device) if device is not None else ref.device
dtype = dtype if dtype is not None else ref.dtype
latent_dim = latent_dim if latent_dim is not None else dit.latent_dim
if generator is None:
if seed is not None:
torch.manual_seed(seed)
x = torch.randn(gen_frames, latent_dim, device=device, dtype=dtype).unsqueeze(0)
else:
x = (
torch.randn(gen_frames, latent_dim, generator=generator, device=generator.device, dtype=dtype)
.to(device)
.unsqueeze(0)
)
if t_grid is not None:
t = torch.tensor(t_grid, device=device, dtype=torch.float32)
else:
t = torch.linspace(0, 1, nfe + 1, device=device, dtype=torch.float32)
if sway_sampling_coef is not None:
t = t + sway_sampling_coef * (torch.cos(math.pi / 2 * t) - 1 + t)
guided = cfg_strength >= 1e-5
if t.numel() < 2 or not bool(torch.all(t[1:] > t[:-1])):
raise ValueError(
f"AuK sampling needs a strictly increasing time grid with at least two points; got {t.tolist()}"
)
try:
for i in range(t.shape[0] - 1):
if guided:
pred = dit(
x,
text,
t[i],
c_mask=c_mask,
ref=ref,
ref_mask=ref_mask,
cfg_infer=True,
cache=True,
)
v_cond, v_uncond = pred.chunk(2, dim=0)
v = v_cond + (v_cond - v_uncond) * cfg_strength
else:
v = dit(x, text, t[i], c_mask=c_mask, ref=ref, ref_mask=ref_mask)
x = x + (t[i + 1] - t[i]) * v
finally:
# The cached text projections belong to this request only; a failed
# step must not leak them into the next one.
dit.clear_cache()
return x
+505
View File
@@ -0,0 +1,505 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
#
# Adapted from the AuK reference release (MIT), which in turn adapts NVIDIA
# BigVGAN (MIT), jik876/hifi-gan (MIT), junjun3518/alias-free-torch
# (Apache-2.0), EdwardDixon/snake (MIT) and adefossez/julius (MIT).
"""BigVGAN-flow VAE for AuK: 24 kHz mono waveform <-> 50 Hz, 64-channel latents.
Inference only. The reference module also carries a residual coupling flow, a KL
term and a discriminator; all three are training-only, so the released
``flow.*`` tensors are skipped (see :meth:`AuKVAE.load_weights`).
Parameter and buffer names follow the reference release, so the published
``vae.safetensors`` loads without a key map. That is also why the convolutions
are built with the legacy ``weight_norm`` parametrization (``weight_g`` /
``weight_v``): :meth:`remove_weight_norm` folds them into plain weights once
the checkpoint is in.
"""
from __future__ import annotations
import inspect
import math
import warnings
from pathlib import Path
import torch
import torch.nn.functional as F
from safetensors.torch import load_file
from torch import nn
from torch.nn.utils import remove_weight_norm as _fold_weight_norm
from torch.nn.utils import weight_norm as _apply_weight_norm
from vllm.logger import init_logger
logger = init_logger(__name__)
__all__ = ["AuKVAE"]
# Checkpoint prefix owned by the training-only coupling flow.
TRAINING_ONLY_PREFIX = "flow."
def _weight_norm(module: nn.Module) -> nn.Module:
"""Legacy ``weight_g``/``weight_v`` split, which is how the release stores these tensors."""
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message=".*weight_norm.*deprecated.*", category=FutureWarning)
return _apply_weight_norm(module)
def _same_padding(kernel_size: int, dilation: int = 1) -> int:
return int((kernel_size * dilation - dilation) / 2)
def _kaiser_sinc_filter(cutoff: float, half_width: float, kernel_size: int) -> torch.Tensor:
"""Kaiser-windowed sinc low-pass, returned as a ``[1, 1, kernel_size]`` conv kernel."""
half_size = kernel_size // 2
attenuation = 2.285 * (half_size - 1) * math.pi * (4 * half_width) + 7.95
if attenuation > 50.0:
beta = 0.1102 * (attenuation - 8.7)
elif attenuation >= 21.0:
beta = 0.5842 * (attenuation - 21.0) ** 0.4 + 0.07886 * (attenuation - 21.0)
else:
beta = 0.0
window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)
if kernel_size % 2 == 0:
time = torch.arange(-half_size, half_size) + 0.5
else:
time = torch.arange(kernel_size) - half_size
if cutoff == 0:
return torch.zeros_like(time).view(1, 1, kernel_size)
taps = 2 * cutoff * window * torch.sinc(2 * cutoff * time)
# Normalize to unit sum so a constant input passes through unscaled.
return (taps / taps.sum()).view(1, 1, kernel_size)
class Conv(nn.Conv1d):
"""1-D convolution over ``[B, C, T]``; ``causal`` swaps centred padding for a left pad."""
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: int = 1,
stride: int = 1,
dilation: int = 1,
bias: bool = True,
causal: bool = False,
) -> None:
super().__init__(
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=0 if causal else _same_padding(kernel_size, dilation),
dilation=dilation,
bias=bias,
)
self.causal = causal
self.left_padding = dilation * (kernel_size - 1) if causal else 0
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.causal:
x = F.pad(x, (self.left_padding, 0))
return super().forward(x)
class ConvTranspose(nn.ConvTranspose1d):
"""Transposed 1-D convolution; ``causal`` drops the trailing ``stride`` samples instead of padding."""
def __init__(
self, in_channels: int, out_channels: int, kernel_size: int, stride: int, causal: bool = False
) -> None:
if causal and kernel_size != 2 * stride:
raise ValueError(f"causal ConvTranspose needs kernel_size == 2 * stride, got {kernel_size} and {stride}")
super().__init__(
in_channels,
out_channels,
kernel_size,
stride=stride,
padding=0 if causal else (kernel_size - stride) // 2,
)
self.causal = causal
self.trim = stride
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = super().forward(x)
return x[:, :, : -self.trim] if self.causal else x
class NormConv(nn.Module):
"""Weight-normalized conv held under ``layer``; the attribute name is load-bearing for the checkpoint."""
def __init__(self, in_channels: int, out_channels: int, kernel_size: int = 1, stride: int = 1) -> None:
super().__init__()
self.layer = _weight_norm(
nn.Conv1d(in_channels, out_channels, kernel_size, stride=stride, padding=(kernel_size - 1) // 2)
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.layer(x)
class ResStack(nn.Module):
"""Stack of dilated residual conv pairs. The leaky slope is torch's 0.01 default, as upstream."""
def __init__(self, channels: int, kernel_size: int = 3, base: int = 3, nums: int = 4) -> None:
super().__init__()
self.layers = nn.ModuleList(
[
nn.Sequential(
nn.LeakyReLU(),
_weight_norm(nn.Conv1d(channels, channels, kernel_size, dilation=base**i, padding=base**i)),
nn.LeakyReLU(),
_weight_norm(nn.Conv1d(channels, channels, kernel_size, dilation=1, padding=1)),
)
for i in range(nums)
]
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
for layer in self.layers:
x = x + layer(x)
return x
class Encoder(nn.Module):
"""Strided conv encoder over ``[B, 1, T]``, emitting ``[B, 2 * latent_dim, T / hop]`` mean/log-std pairs."""
def __init__(
self,
latent_dim: int,
channels: tuple[int, ...],
down_sample_factors: tuple[int, ...],
in_channels: int = 1,
proj_kernel_size: int = 3,
stack_kernel_size: int = 3,
stack_dilation_base: int = 2,
stacks: int = 6,
) -> None:
super().__init__()
layers: list[nn.Module] = [
NormConv(in_channels, channels[0], kernel_size=proj_kernel_size),
nn.LeakyReLU(0.2, True),
]
for in_c, out_c, factor in zip(channels[:-1], channels[1:], down_sample_factors, strict=True):
layers += [
NormConv(in_c, out_c, kernel_size=factor * 2, stride=factor),
ResStack(out_c, stack_kernel_size, stack_dilation_base, stacks),
nn.LeakyReLU(0.2, True),
]
layers.append(NormConv(channels[-1], latent_dim * 2, kernel_size=proj_kernel_size))
self.generator = nn.Sequential(*layers)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.generator(x)
class SnakeBeta(nn.Module):
"""``x + sin^2(alpha * x) / beta`` with per-channel learned alpha and beta (log-scale in AuK)."""
def __init__(self, channels: int, alpha_logscale: bool = False) -> None:
super().__init__()
self.alpha_logscale = alpha_logscale
init = torch.zeros(channels) if alpha_logscale else torch.ones(channels)
self.alpha = nn.Parameter(init.clone())
self.beta = nn.Parameter(init.clone())
self.eps = 1e-9
def forward(self, x: torch.Tensor) -> torch.Tensor:
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
beta = self.beta.unsqueeze(0).unsqueeze(-1)
if self.alpha_logscale:
alpha = torch.exp(alpha)
beta = torch.exp(beta)
return x + (1.0 / (beta + self.eps)) * torch.sin(x * alpha).pow(2)
class LowPass(nn.Module):
"""Fixed FIR low-pass with replicate padding, applied per channel."""
def __init__(
self,
cutoff: float = 0.5,
half_width: float = 0.6,
stride: int = 1,
kernel_size: int = 12,
causal: bool = False,
) -> None:
super().__init__()
if not 0.0 <= cutoff <= 0.5:
raise ValueError(f"cutoff must lie in [0, 0.5], got {cutoff}")
if causal:
self.pad_left, self.pad_right = kernel_size - 1, 0
else:
even = kernel_size % 2 == 0
self.pad_left = kernel_size // 2 - int(even)
self.pad_right = kernel_size // 2
self.stride = stride
self.register_buffer("filter", _kaiser_sinc_filter(cutoff, half_width, kernel_size))
def forward(self, x: torch.Tensor) -> torch.Tensor:
channels = x.size(1)
x = F.pad(x, (self.pad_left, self.pad_right), mode="replicate")
return F.conv1d(x, self.filter.expand(channels, -1, -1), stride=self.stride, groups=channels)
class Upsample(nn.Module):
"""Band-limited interpolation by ``ratio``. Always non-causal, matching how AuK builds it."""
def __init__(self, ratio: int = 2, kernel_size: int = 12) -> None:
super().__init__()
self.ratio = ratio
self.pad = kernel_size // ratio - 1
self.pad_left = self.pad * ratio + (kernel_size - ratio) // 2
self.pad_right = self.pad * ratio + (kernel_size - ratio + 1) // 2
self.register_buffer("filter", _kaiser_sinc_filter(0.5 / ratio, 0.6 / ratio, kernel_size))
def forward(self, x: torch.Tensor) -> torch.Tensor:
channels = x.size(1)
x = F.pad(x, (self.pad, self.pad), mode="replicate")
x = self.ratio * F.conv_transpose1d(x, self.filter.expand(channels, -1, -1), stride=self.ratio, groups=channels)
return x[..., self.pad_left : -self.pad_right]
class Downsample(nn.Module):
"""Decimation by ``ratio`` behind a low-pass; ``lowpass`` holds the filter buffer."""
def __init__(self, ratio: int = 2, kernel_size: int = 12, causal: bool = False) -> None:
super().__init__()
self.lowpass = LowPass(
cutoff=0.5 / ratio, half_width=0.6 / ratio, stride=ratio, kernel_size=kernel_size, causal=causal
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.lowpass(x)
class AliasFreeActivation(nn.Module):
"""Oversample, apply the pointwise activation, decimate, so snake harmonics do not fold back."""
def __init__(
self,
activation: nn.Module,
up_ratio: int = 2,
down_ratio: int = 2,
up_kernel_size: int = 12,
down_kernel_size: int = 12,
causal: bool = False,
) -> None:
super().__init__()
self.act = activation
self.upsample = Upsample(up_ratio, up_kernel_size)
self.downsample = Downsample(down_ratio, down_kernel_size, causal=causal)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.downsample(self.act(self.upsample(x)))
class AmpBlock(nn.Module):
"""Anti-aliased multi-periodicity residual block: alternating alias-free snake and dilated convs."""
def __init__(
self,
channels: int,
kernel_size: int,
dilations: tuple[int, ...],
snake_logscale: bool = True,
causal: bool = True,
act_causal: bool = False,
) -> None:
super().__init__()
self.convs1 = nn.ModuleList(
[_weight_norm(Conv(channels, channels, kernel_size, 1, dilation=d, causal=causal)) for d in dilations]
)
self.convs2 = nn.ModuleList(
[_weight_norm(Conv(channels, channels, kernel_size, 1, dilation=1, causal=causal)) for _ in dilations]
)
self.activations = nn.ModuleList(
[
AliasFreeActivation(SnakeBeta(channels, alpha_logscale=snake_logscale), causal=act_causal)
for _ in range(len(self.convs1) + len(self.convs2))
]
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
acts1, acts2 = self.activations[::2], self.activations[1::2]
for conv1, conv2, act1, act2 in zip(self.convs1, self.convs2, acts1, acts2, strict=True):
x = conv2(act2(conv1(act1(x)))) + x
return x
class AuKVAE(nn.Module):
"""AuK's audio codec: :meth:`encode` waveform to normalized latents, :meth:`decode` back.
Build from the released ``model.vae.model_init_kwargs`` with :meth:`from_config`, then
:meth:`load_weights`. Latents are normalized by the checkpoint's global statistics, which is
the space the rectified-flow transformer works in.
"""
def __init__(
self,
*,
upsample_rates: tuple[int, ...] = (5, 4, 3, 2, 2, 2),
upsample_kernel_sizes: tuple[int, ...] = (10, 8, 6, 4, 4, 4),
upsample_initial_channel: int = 1536,
resblock_kernel_sizes: tuple[int, ...] = (3, 7, 11),
resblock_dilation_sizes: tuple[tuple[int, ...], ...] = ((1, 3, 5), (1, 3, 5), (1, 3, 5)),
downsample_rates: tuple[int, ...] = (2, 2, 2, 3, 4, 5),
downsample_channels: tuple[int, ...] = (12, 24, 48, 96, 192, 384, 768),
snake_logscale: bool = True,
latent_dim: int = 64,
use_vae: bool = True,
causal: bool = True,
act_causal: bool = True,
sample_rate: int = 24000,
) -> None:
super().__init__()
if not use_vae:
raise ValueError("AuKVAE requires use_vae=True: the encoder head emits mean/log-std pairs")
self.latent_dim = latent_dim
self.sample_rate = sample_rate
self.hop_size = math.prod(downsample_rates)
self.num_kernels = len(resblock_kernel_sizes)
self.num_upsamples = len(upsample_rates)
self._norm_folded = False
self.register_buffer("global_mean", torch.zeros(latent_dim, dtype=torch.float32))
self.register_buffer("global_log_std", torch.ones(latent_dim, dtype=torch.float32))
self.audio_encoder = Encoder(
latent_dim=latent_dim,
channels=tuple(downsample_channels),
down_sample_factors=tuple(downsample_rates),
)
self.conv_pre = _weight_norm(Conv(latent_dim, upsample_initial_channel, 7, 1, causal=False))
self.ups = nn.ModuleList()
for i, (rate, kernel) in enumerate(zip(upsample_rates, upsample_kernel_sizes, strict=True)):
# One-element inner list: the checkpoint keys are ``ups.<i>.0.*``.
self.ups.append(
nn.ModuleList(
[
_weight_norm(
ConvTranspose(
upsample_initial_channel // (2**i),
upsample_initial_channel // (2 ** (i + 1)),
kernel,
rate,
causal=causal,
)
)
]
)
)
self.resblocks = nn.ModuleList()
channels = upsample_initial_channel
for i in range(self.num_upsamples):
channels = upsample_initial_channel // (2 ** (i + 1))
for kernel, dilations in zip(resblock_kernel_sizes, resblock_dilation_sizes, strict=True):
self.resblocks.append(
AmpBlock(
channels,
kernel,
tuple(dilations),
snake_logscale=snake_logscale,
causal=causal,
act_causal=act_causal,
)
)
self.activation_post = AliasFreeActivation(
SnakeBeta(channels, alpha_logscale=snake_logscale), causal=act_causal
)
self.conv_post = _weight_norm(Conv(channels, 1, 7, 1, causal=causal, bias=False))
@classmethod
def from_config(cls, cfg: dict) -> AuKVAE:
"""Build from the released ``model_init_kwargs`` mapping, ignoring training-only entries."""
accepted = set(inspect.signature(cls.__init__).parameters) - {"self"}
return cls(**{key: value for key, value in cfg.items() if key in accepted})
def load_weights(
self, path: str | Path, *, device: str | torch.device | None = None, fold_weight_norm: bool = True
) -> tuple[list[str], list[str]]:
"""Load a released ``vae.safetensors``, then fold weight norm away. Returns (missing, unexpected).
``device`` defaults to where the module already lives, which is the placement that matters:
the fold reduces ``weight_v`` to a norm, and a CPU reduction lands up to one ULP away from the
accelerator's. The encoder amplifies its activations roughly fiftyfold, so that one ULP grows
into a ~4e-4 relative drift by the latent head. Move the module first, then load.
"""
target = self.global_mean.device if device is None else torch.device(device)
missing, unexpected = self.load_state_dict(load_file(str(path), device=str(target)), strict=False)
missing, unexpected = list(missing), list(unexpected)
stray = [key for key in missing + unexpected if not key.startswith(TRAINING_ONLY_PREFIX)]
if stray:
raise ValueError(f"AuK VAE weights at {path} do not match this module: {sorted(stray)[:8]}")
logger.info(
"Loaded AuK VAE from %s (skipped %d training-only %s* tensors)", path, len(unexpected), TRAINING_ONLY_PREFIX
)
if fold_weight_norm:
self.remove_weight_norm()
return missing, unexpected
def remove_weight_norm(self) -> None:
"""Fold ``weight_g``/``weight_v`` into plain weights, on whichever device the module is on.
Idempotent, and meant to run once the checkpoint is in. See :meth:`load_weights` for why the
device this runs on is a numeric decision rather than a convenience.
"""
if self._norm_folded:
return
folded = 0
for module in self.modules():
if hasattr(module, "weight_g"):
_fold_weight_norm(module)
folded += 1
self._norm_folded = True
logger.debug("Folded weight norm on %d AuK VAE convolutions", folded)
def encode(
self, wav: torch.Tensor, *, sample: bool = False, generator: torch.Generator | None = None
) -> torch.Tensor:
"""Encode a mono ``[1, T]`` waveform at :attr:`sample_rate` into ``[1, T / hop_size, latent_dim]``.
``sample=False`` takes the posterior mean, which is what a reproducible pipeline wants;
``sample=True`` draws ``mean + randn * exp(log_std)`` from ``generator``. Latents come back
normalized by the checkpoint's global statistics.
"""
if wav.dim() != 2 or wav.size(0) != 1:
raise ValueError(f"encode expects a mono [1, T] waveform, got {tuple(wav.shape)}")
# Statistics stay in fp32 even under an ambient autocast, as in the reference.
with torch.autocast(device_type=wav.device.type, enabled=False):
mean, log_std = self.audio_encoder(wav.unsqueeze(1)).chunk(2, dim=1)
if sample:
latents = mean + self._noise_like(mean, generator) * torch.exp(log_std)
else:
latents = mean
latents = latents.transpose(1, 2).float()
return (latents - self.global_mean.float()) / torch.sqrt(self.global_log_std.float())
def decode(self, latents: torch.Tensor) -> torch.Tensor:
"""Decode normalized ``[B, Np, latent_dim]`` latents into a ``[B, Np * hop_size]`` waveform in [-1, 1]."""
if latents.dim() != 3 or latents.size(-1) != self.latent_dim:
raise ValueError(f"decode expects [B, Np, {self.latent_dim}] latents, got {tuple(latents.shape)}")
x = latents.float() * torch.sqrt(self.global_log_std.float()) + self.global_mean.float()
x = self.conv_pre(x.permute(0, 2, 1))
for i in range(self.num_upsamples):
for up in self.ups[i]:
x = up(x)
first = i * self.num_kernels
residual = self.resblocks[first](x)
for j in range(1, self.num_kernels):
residual = residual + self.resblocks[first + j](x)
x = residual / self.num_kernels
x = self.conv_post(self.activation_post(x))
return torch.clamp(x, min=-1.0, max=1.0).squeeze(1)
@staticmethod
def _noise_like(reference: torch.Tensor, generator: torch.Generator | None) -> torch.Tensor:
"""Draw standard normal noise shaped like ``reference``; ``generator`` decides which RNG stream."""
device = reference.device if generator is None else generator.device
noise = torch.randn(reference.shape, generator=generator, dtype=reference.dtype, device=device)
return noise.to(reference.device)
@@ -0,0 +1,486 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""AuK audio-editing pipeline for vLLM-Omni.
Turns a layer-fused text condition (produced by the encoder stage) plus an
optional source clip into a 24 kHz waveform: the source clip is encoded to VAE
latents, a rectified-flow DiT integrates the target latents conditioned on
both, and the BigVGAN-flow decoder renders the waveform.
"""
from __future__ import annotations
import json
import os
from collections.abc import Iterable
from typing import Any, ClassVar
import numpy as np
import torch
import torchaudio
from safetensors import safe_open
from torch import nn
from vllm.logger import init_logger
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
from vllm_omni.diffusion.models.interface import (
SupportAudioInput,
SupportAudioOutput,
SupportsComponentDiscovery,
)
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.model_extras.auk import resolve_gen_frames
logger = init_logger(__name__)
# Longest ref + target latent sequence the released checkpoints accept.
MAX_LATENT_FRAMES = 65536
# Weight-file layout: the DiT lives under this prefix; the layer-fusion
# parameters sit next to it but belong to the encoder stage.
_DIT_PREFIX = "transformer."
_FUSION_KEYS = frozenset({"layer_weights", "layer_scale"})
# The distilled variant only reproduces its training recipe at these settings.
_FLASH_NFE = 4
_FLASH_CFG = 0.0
# Decorrelates the VAE posterior draw from the initial latent for the same request seed.
_VAE_SEED_OFFSET = 0x5EED_0A0C
def get_auk_post_process_func(od_config: OmniDiffusionConfig):
"""Create the post-processing function for AuK audio output.
Tensor output types pass through; anything else becomes a numpy waveform.
The sample rate is not attached here: the output formatter reads
``AuKPipeline.audio_sample_rate`` for audio-output pipelines.
"""
del od_config # The conversion does not depend on the config.
def post_process_func(
audio: torch.Tensor,
output_type: str = "np",
):
if output_type in ("pt", "latent"):
return audio
return audio.cpu().float().numpy()
return post_process_func
def _prompt_mapping(prompt: Any) -> dict[str, Any]:
"""Return the prompt dict, or an empty mapping for a bare text prompt."""
if isinstance(prompt, dict):
return prompt
return {}
def _unwrap_single(value: Any) -> Any:
"""Unwrap the one-element lists the serving layer sometimes produces."""
if isinstance(value, list) and len(value) == 1:
return value[0]
return value
def _split_audio(audio: Any, default_sample_rate: int) -> tuple[Any, int]:
"""Split a multimodal audio item into raw samples and a sample rate."""
if isinstance(audio, (tuple, list)) and len(audio) == 2 and isinstance(audio[1], (int, float)):
return audio[0], int(audio[1])
return audio, default_sample_rate
class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComponentDiscovery):
"""Instruction-driven audio generation and editing with AuK.
One request per forward: the rectified-flow ODE runs over the whole target
span and the batch dimension carries the CFG branches, so there is nothing
to share between requests yet.
Args:
od_config: OmniDiffusion configuration. ``od_config.model`` must be an
assembled AuK directory (``config.json``, ``auk.safetensors``,
``vae.safetensors``), as produced by
``tools/prepare_auk_checkpoint.py``.
prefix: Unused; kept for the pipeline construction contract.
"""
supports_request_batch = False
# Picked up by ``supports_audio_output`` in the diffusion engine so the
# default stage metadata reports ``final_output_type="audio"`` and the
# ``multimodal_output`` payload includes the sample rate.
support_audio_output: ClassVar[bool] = True
support_audio_input: ClassVar[bool] = True
audio_sample_rate: ClassVar[int] = 24000
# The DiT cannot run without an encoder-stage text condition, which the
# engine's synthetic warmup request has no way to produce.
dummy_run_num_frames: ClassVar[int] = 0
_dit_modules: ClassVar[list[str]] = ["dit"]
_encoder_modules: ClassVar[list[str]] = []
_vae_modules: ClassVar[list[str]] = ["vae"]
def __init__(
self,
*,
od_config: OmniDiffusionConfig,
prefix: str = "",
) -> None:
super().__init__()
del prefix # Weights are not namespaced: one checkpoint, one pipeline.
self.od_config = od_config
self.device = get_local_device()
self.dtype = getattr(od_config, "dtype", None) or torch.bfloat16
model_dir = od_config.model
if not model_dir or not os.path.isdir(model_dir):
raise ValueError(
f"AuK needs an assembled local checkpoint directory, got {model_dir!r}. "
"Build one with tools/prepare_auk_checkpoint.py."
)
config = _read_config(model_dir)
vae_config = dict(config["vae"])
self.latent_dim = int(vae_config["latent_dim"])
self.hop_size = int(vae_config["downsample_rate"])
self.sample_rate = int(vae_config["target_sample_rate"])
self.variant = str(config.get("variant", "base"))
self.is_flash = self.variant == "flash"
self.flash_t_grid = [float(t) for t in config.get("flash_t_grid") or ()]
if self.is_flash and len(self.flash_t_grid) != _FLASH_NFE + 1:
raise ValueError(f"The flash variant needs a {_FLASH_NFE + 1}-point flash_t_grid, got {self.flash_t_grid}.")
defaults = dict(config.get("defaults") or {})
self.default_nfe = int(defaults.get("nfe", 32))
self.default_cfg = float(defaults.get("cfg", 2.0))
self.default_sway = defaults.get("sway", -1.0)
self._flash_lock_logged = False
# The codec is small and numerically sensitive: it stays in fp32. It
# must sit on the inference device BEFORE load_weights folds weight
# norm: a CPU fold lands one ULP off the accelerator's and the encoder
# amplifies that into a 1e-2 latent drift.
self.vae = AuKVAE.from_config(vae_config["model_init_kwargs"]).to(device=self.device, dtype=torch.float32)
self.vae.load_weights(os.path.join(model_dir, "vae.safetensors"))
self.vae = self.vae.eval()
self.vae.requires_grad_(False)
self._check_vae_geometry()
# The transformer owns every key the checkpoint tool writes into `dit`,
# `attn_mask_enabled` included, so the section passes straight through.
self.dit = AuKTransformer(latent_dim=self.latent_dim, **config["dit"])
self.dit = self.dit.to(dtype=self.dtype)
self.dit.load_state_dict(_read_dit_weights(model_dir, self.dtype), strict=True)
self.dit = self.dit.to(device=self.device).eval()
self.dit.requires_grad_(False)
logger.info(
"AuK pipeline ready: variant=%s dtype=%s latent_dim=%d hop=%d sample_rate=%d",
self.variant,
self.dtype,
self.latent_dim,
self.hop_size,
self.sample_rate,
)
# The assembled checkpoint is not a diffusers layout: __init__ reads
# auk.safetensors and vae.safetensors directly, so the loader has no
# component sources to stream and load_weights only reports what is loaded.
weights_sources: ClassVar[tuple] = ()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stray = [name for name, _ in weights]
if stray:
logger.warning("AuKPipeline ignores %d loader-provided weights (e.g. %s)", len(stray), stray[:3])
return {name for name, _ in self.named_parameters()}
def _check_vae_geometry(self) -> None:
"""Fail fast when config.json and the VAE disagree on latent geometry."""
declared = {
"latent_dim": (self.latent_dim, int(self.vae.latent_dim)),
"downsample_rate": (self.hop_size, int(self.vae.hop_size)),
"target_sample_rate": (self.sample_rate, int(self.vae.sample_rate)),
}
bad = {name: pair for name, pair in declared.items() if pair[0] != pair[1]}
if bad:
raise ValueError(f"config.json and the AuK VAE disagree (config, vae): {bad}")
if self.sample_rate != self.audio_sample_rate:
raise ValueError(
f"AuK advertises {self.audio_sample_rate} Hz output but the checkpoint is {self.sample_rate} Hz."
)
def _resolve_generator(self, sampling_params: Any) -> torch.Generator | None:
"""Per-request generator; the process-global RNG is never seeded."""
generator = sampling_params.generator
if isinstance(generator, list):
if len(generator) > 1:
logger.warning(
"AuKPipeline runs one request per forward; using the first of %d generators", len(generator)
)
generator = generator[0] if generator else None
if generator is None and sampling_params.seed is not None:
generator = torch.Generator(device=self.device).manual_seed(int(sampling_params.seed))
return generator
def _prepare_waveform(self, audio: Any) -> torch.Tensor:
"""Return mono float32 samples ``[1, T]`` at the VAE sample rate.
``T`` is truncated to a whole number of latent frames. The encoder is
causal, so dropping a trailing partial frame leaves every emitted frame
unchanged.
"""
data, sample_rate = _split_audio(audio, self.sample_rate)
if isinstance(data, np.ndarray):
wav = torch.from_numpy(np.ascontiguousarray(data))
else:
wav = torch.as_tensor(data)
wav = wav.detach().to(device="cpu", dtype=torch.float32)
if wav.ndim == 1:
wav = wav.unsqueeze(0)
elif wav.ndim == 2:
# Accept both [channels, samples] and soundfile's [samples, channels];
# the channel axis is the short one (at most 8 wide).
if wav.shape[0] > wav.shape[1] and wav.shape[1] <= 8:
wav = wav.transpose(0, 1)
if wav.shape[0] > 1:
wav = wav.mean(dim=0, keepdim=True)
else:
raise ValueError(f"AuK audio input must be [samples] or [channels, samples], got {tuple(wav.shape)}.")
if wav.shape[-1] == 0:
raise ValueError("AuK audio input is empty.")
if not torch.isfinite(wav).all():
raise ValueError("AuK audio input contains NaN or Inf.")
if sample_rate != self.sample_rate:
wav = torchaudio.functional.resample(wav, sample_rate, self.sample_rate)
frames = wav.shape[-1] // self.hop_size
if frames < 1:
raise ValueError(
f"AuK audio input is shorter than one latent frame ({self.hop_size} samples at {self.sample_rate} Hz)."
)
return wav[..., : frames * self.hop_size]
def _encode_source(
self,
audio: Any,
*,
sample: bool,
generator: torch.Generator | None,
) -> torch.Tensor:
"""Encode the source clip to normalized latents ``[1, np, latent_dim]``."""
if audio is None:
return torch.zeros(1, 0, self.latent_dim, device=self.device, dtype=torch.float32)
wav = self._prepare_waveform(_unwrap_single(audio)).to(self.device)
return self.vae.encode(wav, sample=sample, generator=generator)
def _target_frames(self, gen_seconds: Any, ref_frames: int) -> int:
"""Target latent length, capped by the remaining context."""
frames = resolve_gen_frames(
None if gen_seconds is None else float(gen_seconds),
ref_frames,
sample_rate=self.sample_rate,
hop=self.hop_size,
)
budget = MAX_LATENT_FRAMES - ref_frames
if budget < 1:
raise ValueError(
f"The source clip already fills the {MAX_LATENT_FRAMES}-frame context ({ref_frames} frames)."
)
if frames > budget:
logger.warning("AuK target of %d frames exceeds the remaining context; clamping to %d.", frames, budget)
frames = budget
return frames
def _resolve_schedule(
self,
sampling_params: Any,
knobs: dict[str, Any],
) -> tuple[int, float, float | None, list[float] | None]:
"""Resolve (nfe, cfg, sway, t_grid), locking the distilled recipe on Flash.
A knob present but ``None`` counts as unspecified: the stage input
processor fills every key it knows about, whether the caller set it or
not.
"""
steps = sampling_params.num_inference_steps
nfe = self.default_nfe if steps is None else int(steps)
if nfe < 1:
raise ValueError(f"num_inference_steps must be >= 1 for AuK; got {steps}")
cfg = float(sampling_params.guidance_scale) if sampling_params.guidance_scale_provided else self.default_cfg
sway = knobs.get("sway")
sway = self.default_sway if sway is None else float(sway)
raw_grid = knobs.get("t_grid")
t_grid = [float(t) for t in raw_grid] if raw_grid else None
if not self.is_flash:
return nfe, cfg, sway, t_grid
locked = (_FLASH_NFE, _FLASH_CFG, None, self.flash_t_grid)
if (nfe, cfg, sway, t_grid) != locked and not self._flash_lock_logged:
self._flash_lock_logged = True
logger.warning(
"AuK-Flash ignores per-request sampling knobs (asked nfe=%s cfg=%s sway=%s t_grid=%s); "
"using the distilled recipe nfe=%d cfg=%s with the checkpoint time grid.",
nfe,
cfg,
sway,
t_grid,
_FLASH_NFE,
_FLASH_CFG,
)
return locked
def _dit_autocast(self):
"""Match the reference, which runs the DiT under bf16 autocast."""
return torch.autocast(
device_type=self.device.type,
dtype=self.dtype,
enabled=self.dtype in (torch.bfloat16, torch.float16),
)
def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]:
"""Generate one waveform.
Args:
req: Request batch holding exactly one request. The prompt carries
``prompt_embeds`` (the fused text condition ``[nt, 2048]``), an
optional ``multi_modal_data["audio"]`` source clip, and
``additional_information["auk"]`` with ``gen_seconds``,
``sway``, ``t_grid`` and ``vae_sample``. ``num_inference_steps``,
``guidance_scale`` and ``seed``/``generator`` come from the
sampling params.
Returns:
One ``DiffusionOutput`` whose ``output`` is a float32 mono waveform
``[T]`` at 24 kHz, or the normalized target latents
``[1, gen_frames, latent_dim]`` when ``output_type`` is ``latent``.
"""
assert req.num_reqs == 1, f"AuKPipeline runs one request per forward, got {req.num_reqs}."
prompt = _prompt_mapping(req.prompts[0])
sampling_params = req.sampling_params
knobs = dict((prompt.get("additional_information") or {}).get("auk") or {})
text = _unwrap_single(prompt.get("prompt_embeds"))
if text is None:
raise ValueError("AuK stage 1 needs `prompt_embeds`, the fused text condition from the encoder stage.")
text = torch.as_tensor(text).to(device=self.device, dtype=self.dtype)
if text.ndim == 3 and text.shape[0] == 1:
text = text[0]
if text.ndim != 2:
raise ValueError(f"AuK `prompt_embeds` must be [nt, text_hidden_dim], got {tuple(text.shape)}.")
text = text.unsqueeze(0)
c_mask = torch.ones(text.shape[:2], dtype=torch.bool, device=self.device)
generator = self._resolve_generator(sampling_params)
audio = (prompt.get("multi_modal_data") or {}).get("audio")
output_type = sampling_params.output_type or "np"
with torch.inference_mode():
# The VAE posterior draw gets its own generator, seeded with a
# fixed offset from the request seed: the initial latent below then
# still equals the reference's fresh manual_seed draw, and the two
# noise streams are not copies of each other.
vae_generator = None
if knobs.get("vae_sample") and generator is not None:
vae_seed = (int(generator.initial_seed()) + _VAE_SEED_OFFSET) % (2**63)
vae_generator = torch.Generator(device=self.device).manual_seed(vae_seed)
ref = self._encode_source(audio, sample=bool(knobs.get("vae_sample")), generator=vae_generator)
ref_frames = ref.shape[1]
ref_mask = torch.ones(ref.shape[:2], dtype=torch.bool, device=self.device)
gen_frames = self._target_frames(knobs.get("gen_seconds"), ref_frames)
nfe, cfg, sway, t_grid = self._resolve_schedule(sampling_params, knobs)
with self._dit_autocast():
latents = sample_latents(
self.dit,
text=text,
c_mask=c_mask,
ref=ref,
ref_mask=ref_mask,
gen_frames=gen_frames,
nfe=nfe,
cfg_strength=cfg,
sway_sampling_coef=sway,
t_grid=t_grid,
seed=None,
latent_dim=self.latent_dim,
device=self.device,
dtype=torch.float32,
generator=generator,
)
latents = latents.float()
if not torch.isfinite(latents).all():
raise RuntimeError("AuK generated latents contain NaN or Inf.")
if output_type == "latent":
return [DiffusionOutput(output=latents.detach().cpu())]
wav = self.vae.decode(latents)
# One mono waveform per request; the formatter expects [T].
wav = wav.detach().to(device="cpu", dtype=torch.float32).reshape(-1)
if not torch.isfinite(wav).all():
raise RuntimeError("AuK generated audio contains NaN or Inf.")
return [DiffusionOutput(output=wav)]
def _read_config(model_dir: str) -> dict[str, Any]:
"""Read the assembled AuK ``config.json``."""
path = os.path.join(model_dir, "config.json")
try:
with open(path) as handle:
config = json.load(handle)
except FileNotFoundError as exc:
raise ValueError(f"AuK checkpoint is missing config.json: {path}") from exc
except json.JSONDecodeError as exc:
logger.error("AuK config.json is not valid JSON: %s", path)
raise ValueError(f"AuK config.json is not valid JSON: {path}") from exc
missing = [key for key in ("dit", "vae") if key not in config]
if missing:
raise ValueError(f"AuK config.json is missing required sections {missing}: {path}")
return config
def _read_dit_weights(model_dir: str, dtype: torch.dtype) -> dict[str, torch.Tensor]:
"""Read the DiT state dict from ``auk.safetensors``, cast to ``dtype``.
Tensors are cast one at a time so the fp32 checkpoint is never fully
resident alongside the model.
"""
path = os.path.join(model_dir, "auk.safetensors")
if not os.path.isfile(path):
raise ValueError(f"AuK checkpoint is missing auk.safetensors: {path}")
with safe_open(path, framework="pt", device="cpu") as checkpoint:
keys = list(checkpoint.keys())
state = dit_state_dict(((key, checkpoint.get_tensor(key).to(dtype)) for key in keys), _DIT_PREFIX)
if not state:
raise ValueError(f"AuK checkpoint has no '{_DIT_PREFIX}*' weights: {path}")
unexpected = sorted(key for key in keys if key not in _FUSION_KEYS and not key.startswith(_DIT_PREFIX))
if unexpected:
logger.warning("Ignoring %d unrecognized keys in %s, e.g. %s", len(unexpected), path, unexpected[:5])
return state