[Diffusion] [Model] Support AudioX (#2077)

Signed-off-by: Zhang Jian <jianmusings@gmail.com>
Signed-off-by: Zhang <jianmusings@gmail.com>
Signed-off-by: Zhang <zhang.jian@u.nus.edu>
Signed-off-by: Zhang Jian <e0322744@u.nus.edu>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Co-authored-by: Zhang Jian <e0322744@u.nus.edu>
Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-05-04 10:24:35 -04:00
1 parent c306511512
commit c708aecb18
25 files changed
+2487 -66

No files matched your search

@@ -0,0 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .audiox_transformer import MMDiffusionTransformer
from .pipeline_audiox import AudioXPipeline, get_audiox_post_process_func
__all__ = [
"AudioXPipeline",
"MMDiffusionTransformer",
"get_audiox_post_process_func",
]
@@ -0,0 +1,487 @@
from __future__ import annotations
import logging
from collections.abc import Iterable
from typing import Any
import torch
import torch.nn as nn
from einops import rearrange
from torch.nn import functional as F
from vllm.distributed import (
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
tensor_model_parallel_all_reduce,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
QKVParallelLinear,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader, sharded_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.layers.fourier import GaussianFourierProjection
from vllm_omni.diffusion.layers.rope import RotaryEmbedding
class AudioXCrossAttention(nn.Module):
def __init__(self, dim: int, nheads: int, prefix: str = ""):
super().__init__()
self.dim = dim
self.nheads = nheads
head_dim = dim // nheads
self.head_dim = head_dim
# to_kv bundle weights arrive in (head, dim, VK-index) interleaved order; pipeline's
# load_weights restacks them to [V|K] before MergedColumnParallelLinear consumes them.
self.to_q = ColumnParallelLinear(dim, dim, bias=False, gather_output=False, prefix=f"{prefix}.to_q")
self.to_kv = MergedColumnParallelLinear(
input_size=dim, output_sizes=[dim, dim], bias=False, gather_output=False, prefix=f"{prefix}.to_kv"
)
self.q_norm = AudioXRMSNorm(head_dim)
self.k_norm = AudioXRMSNorm(head_dim)
local_nheads = nheads // get_tensor_model_parallel_world_size()
self.attn = Attention(
num_heads=local_nheads,
head_size=head_dim,
softmax_scale=head_dim**-0.5,
causal=False,
)
def forward(self, x: torch.Tensor, context: torch.Tensor | None = None) -> torch.Tensor:
tp = get_tensor_model_parallel_world_size()
local_h = self.nheads // tp
d = self.head_dim
# ROCm's ``wvSplitK`` GEMM bypasses autocast and rejects mismatched dtypes,
# so pre-match the input to the linear's weight dtype.
weight_dtype = self.to_q.weight.dtype
q_flat, _ = self.to_q(x.to(weight_dtype))
q = rearrange(q_flat, "b n (h d) -> b n h d", h=local_h, d=d)
kv_flat, _ = self.to_kv(context.to(weight_dtype))
v_flat, k_flat = kv_flat.chunk(2, dim=-1)
# Upstream `CrossAttention.forward` unpacks `pre_attention()` as `q, v, k = ...` (K/V
# swapped), AND only normalizes the first chunk of to_kv via k_norm. Trained weights
# depend on this quirk: first chunk normalized -> V; second chunk unnormalized -> K.
v = rearrange(v_flat, "b n (h d) -> b n h d", h=local_h, d=d)
k = rearrange(k_flat, "b n (h d) -> b n h d", h=local_h, d=d)
q = self.q_norm(q)
v = self.k_norm(v)
out = self.attn(q.contiguous(), k.contiguous(), v.contiguous(), attn_metadata=None)
out = rearrange(out, "b n h d -> b n (h d)").contiguous()
if tp > 1:
out = tensor_model_parallel_all_gather(out, dim=-1)
return out
logger = logging.getLogger(__name__)
__all__ = [
"AudioXMMChannelLastConv1d",
"AudioXMMConvFeedForward",
"AudioXMMDiTSelfAttention",
"AudioXMMDiTBlock",
"ContinuousMMDiTTransformer",
"MMDiffusionTransformer",
]
class AudioXMMChannelLastConv1d(nn.Conv1d):
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = rearrange(x, "b n c -> b c n")
x = super().forward(x)
x = rearrange(x, "b c n -> b n c")
return x
class _ColumnParallelChannelLastConv1d(AudioXMMChannelLastConv1d):
def __init__(self, in_channels: int, out_channels_total: int, **kwargs: Any):
tp_size = get_tensor_model_parallel_world_size()
assert out_channels_total % tp_size == 0, (out_channels_total, tp_size)
super().__init__(in_channels, out_channels_total // tp_size, **kwargs)
self.weight.weight_loader = sharded_weight_loader(0)
class _RowParallelChannelLastConv1d(AudioXMMChannelLastConv1d):
def __init__(self, in_channels_total: int, out_channels: int, **kwargs: Any):
tp_size = get_tensor_model_parallel_world_size()
assert in_channels_total % tp_size == 0, (in_channels_total, tp_size)
super().__init__(in_channels_total // tp_size, out_channels, **kwargs)
self._tp_size = tp_size
self.weight.weight_loader = sharded_weight_loader(1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
y = super().forward(x)
if self._tp_size > 1:
y = tensor_model_parallel_all_reduce(y)
return y
class AudioXMMConvFeedForward(nn.Module):
def __init__(
self,
dim: int,
hidden_dim: int,
multiple_of: int = 256,
kernel_size: int = 3,
padding: int = 1,
):
super().__init__()
hidden_dim = int(2 * hidden_dim / 3)
hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
self.w1 = _ColumnParallelChannelLastConv1d(
dim, hidden_dim, bias=False, kernel_size=kernel_size, padding=padding
)
self.w2 = _RowParallelChannelLastConv1d(hidden_dim, dim, bias=False, kernel_size=kernel_size, padding=padding)
self.w3 = _ColumnParallelChannelLastConv1d(
dim, hidden_dim, bias=False, kernel_size=kernel_size, padding=padding
)
def forward(self, x):
return self.w2(F.silu(self.w1(x)) * self.w3(x))
class AudioXRMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
super().__init__()
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
mean_sq = torch.mean(x**2, dim=-1, keepdim=True)
scale = torch.rsqrt(mean_sq + self.eps)
return x * scale
class AudioXMMDiTSelfAttention(nn.Module):
def __init__(self, dim: int, nheads: int, prefix: str = ""):
super().__init__()
self.dim = dim
self.nheads = nheads
head_dim = dim // nheads
self.head_dim = head_dim
# Bundle weights arrive in interleaved (h, d, qkv) layout; pipeline's load_weights
# restacks to [Q|K|V] before QKVParallelLinear's weight_loader consumes them.
self.qkv = QKVParallelLinear(
hidden_size=dim,
head_size=head_dim,
total_num_heads=nheads,
bias=True,
prefix=f"{prefix}.qkv",
)
self.q_norm = AudioXRMSNorm(head_dim)
self.k_norm = AudioXRMSNorm(head_dim)
self.rope = RotaryEmbedding(is_neox_style=False)
self.attn = Attention(
num_heads=self.qkv.num_heads,
head_size=head_dim,
softmax_scale=head_dim**-0.5,
causal=False,
)
def apply_attention(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
out = self.attn(q.contiguous(), k.contiguous(), v.contiguous(), attn_metadata=None)
out = rearrange(out, "b n h d -> b n (h d)").contiguous()
# Downstream linear1/ffn are replicated and expect full hidden dim.
if get_tensor_model_parallel_world_size() > 1:
out = tensor_model_parallel_all_gather(out, dim=-1)
return out
def pre_attention(self, x: torch.Tensor, rot: tuple[torch.Tensor, torch.Tensor] | None = None):
# ROCm GEMM dtype cast — see AudioXCrossAttention.forward.
qkv, _ = self.qkv(x.to(self.qkv.weight.dtype))
local_h = self.qkv.num_heads
d = self.head_dim
q_size = local_h * d
q, k, v = qkv.split([q_size, q_size, q_size], dim=-1)
q = rearrange(q, "b n (h d) -> b n h d", h=local_h, d=d)
k = rearrange(k, "b n (h d) -> b n h d", h=local_h, d=d)
v = rearrange(v, "b n (h d) -> b n h d", h=local_h, d=d)
q = self.q_norm(q)
k = self.k_norm(k)
if rot is not None:
cos, sin = rot
cos = cos.to(dtype=q.dtype)
sin = sin.to(dtype=q.dtype)
q = self.rope(q, cos, sin)
k = self.rope(k, cos, sin)
return q, k, v
def forward(
self,
x: torch.Tensor,
rot: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
q, k, v = self.pre_attention(x, rot=rot)
return self.apply_attention(q, k, v)
class AudioXMMDiTBlock(nn.Module):
def __init__(
self,
dim: int,
nhead: int,
mlp_ratio: float = 4.0,
prefix: str = "",
):
super().__init__()
self.norm1 = nn.LayerNorm(dim, elementwise_affine=False)
self.attn = AudioXMMDiTSelfAttention(dim, nhead, prefix=f"{prefix}.attn")
self.cross_attn = AudioXCrossAttention(dim, nhead, prefix=f"{prefix}.cross_attn")
self.linear1 = AudioXMMChannelLastConv1d(dim, dim, kernel_size=3, padding=1)
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False)
self.ffn = AudioXMMConvFeedForward(dim, int(dim * mlp_ratio), kernel_size=3, padding=1)
self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(dim, 6 * dim, bias=True))
def pre_attention(self, x: torch.Tensor, c: torch.Tensor, rot: tuple[torch.Tensor, torch.Tensor] | None):
modulation = self.adaLN_modulation(c)
(shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp) = modulation.chunk(6, dim=-1)
x = self.norm1(x) * (1 + scale_msa) + shift_msa
q, k, v = self.attn.pre_attention(x, rot)
return (q, k, v), (gate_msa, shift_mlp, scale_mlp, gate_mlp)
def post_attention(self, x: torch.Tensor, attn_out: torch.Tensor, c: tuple[torch.Tensor, ...], context=None):
(gate_msa, shift_mlp, scale_mlp, gate_mlp) = c
x = x + self.linear1(attn_out) * gate_msa
x = x + self.cross_attn(x, context=context)
r = self.norm2(x) * (1 + scale_mlp) + shift_mlp
x = x + self.ffn(r) * gate_mlp
return x
def forward(
self,
x: torch.Tensor,
cond: torch.Tensor,
rot: tuple[torch.Tensor, torch.Tensor] | None,
context: torch.Tensor = None,
) -> torch.Tensor:
x_qkv, x_conditions = self.pre_attention(x, cond, rot)
attn_out = self.attn.apply_attention(*x_qkv)
x = self.post_attention(x, attn_out, x_conditions, context=context)
return x
class MMDiffusionTransformer(nn.Module):
"""AudioX MMDiT, specialized for the published bundle (`zhangj1an/AudioX`).
The bundle fixes patch_size=1, transformer_type="continuous_transformer",
cond_token_dim=768 (>0, project_cond_tokens=False), and never sets
prepend_cond_dim or input_concat_dim, so those code paths are removed.
"""
def __init__(
self,
io_channels: int,
embed_dim: int,
cond_token_dim: int,
global_cond_dim: int,
depth: int,
num_heads: int,
project_cond_tokens: bool = False,
project_global_cond: bool = True,
**kwargs,
):
super().__init__()
if kwargs:
logger.debug("MMDiffusionTransformer ignoring unused config keys: %s", sorted(kwargs.keys()))
if project_cond_tokens:
raise ValueError("AudioX bundle requires project_cond_tokens=False to match official checkpoints.")
self.cond_token_dim = cond_token_dim
timestep_features_dim = 256
self.timestep_features = GaussianFourierProjection(
in_features=1,
embedding_size=timestep_features_dim // 2,
scale=1.0,
trainable=False,
)
self.to_timestep_embed = nn.Sequential(
nn.Linear(timestep_features_dim, embed_dim, bias=True),
nn.SiLU(),
nn.Linear(embed_dim, embed_dim, bias=True),
)
cond_embed_dim = cond_token_dim if not project_cond_tokens else embed_dim
self.to_cond_embed = nn.Sequential(
nn.Linear(cond_token_dim, cond_embed_dim, bias=False),
nn.SiLU(),
nn.Linear(cond_embed_dim, cond_embed_dim, bias=False),
)
# ``to_global_embed`` weights live in the bundle but global conditioning is always None
# at inference; kept so AutoWeightsLoader has a slot to load them into.
global_embed_dim = global_cond_dim if not project_global_cond else embed_dim
self.to_global_embed = nn.Sequential(
nn.Linear(global_cond_dim, global_embed_dim, bias=False),
nn.SiLU(),
nn.Linear(global_embed_dim, global_embed_dim, bias=False),
)
self.transformer = ContinuousMMDiTTransformer(
dim=embed_dim,
depth=depth,
dim_heads=embed_dim // num_heads,
dim_in=io_channels,
dim_out=io_channels,
)
self.preprocess_conv = nn.Conv1d(io_channels, io_channels, 1, bias=False)
nn.init.zeros_(self.preprocess_conv.weight)
self.postprocess_conv = nn.Conv1d(io_channels, io_channels, 1, bias=False)
nn.init.zeros_(self.postprocess_conv.weight)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def _forward(self, x, t, cross_attn_cond):
timestep_embed = self.to_timestep_embed(self.timestep_features(t[:, None]))
prepend_inputs = timestep_embed.unsqueeze(1)
prepend_length = prepend_inputs.shape[1]
x = self.preprocess_conv(x) + x
x = rearrange(x, "b c n -> b n c")
output = self.transformer(x, prepend_embeds=prepend_inputs, context=cross_attn_cond)
output = rearrange(output, "b n c -> b c n")[:, :, prepend_length:]
return self.postprocess_conv(output) + output
def forward(
self,
x,
t,
cross_attn_cond,
negative_cross_attn_cond=None,
negative_cross_attn_mask=None,
cfg_scale: float = 1.0,
scale_phi: float = 0.0,
**kwargs,
):
if cfg_scale == 1.0:
return self._forward(x, t, cross_attn_cond)
# Classifier-free guidance: batch the conditional + unconditional pass.
null_embed = torch.zeros_like(cross_attn_cond)
if negative_cross_attn_cond is not None and negative_cross_attn_mask is not None:
mask = negative_cross_attn_mask.to(torch.bool).unsqueeze(2)
negative_cross_attn_cond = torch.where(mask, negative_cross_attn_cond, null_embed)
uncond = negative_cross_attn_cond if negative_cross_attn_cond is not None else null_embed
batch_output = self._forward(
torch.cat([x, x], dim=0),
torch.cat([t, t], dim=0),
torch.cat([cross_attn_cond, uncond], dim=0),
)
cond_output, uncond_output = torch.chunk(batch_output, 2, dim=0)
cfg_output = uncond_output + (cond_output - uncond_output) * cfg_scale
if scale_phi == 0.0:
return cfg_output
cond_std = cond_output.std(dim=1, keepdim=True)
cfg_std = cfg_output.std(dim=1, keepdim=True)
return scale_phi * (cfg_output * (cond_std / cfg_std)) + (1 - scale_phi) * cfg_output
class ContinuousMMDiTTransformer(nn.Module):
def __init__(
self,
dim,
depth,
*,
dim_in=None,
dim_out=None,
dim_heads=64,
_latent_seq_len=237,
):
super().__init__()
self.dim = dim
self.depth = depth
self.project_in = nn.Linear(dim_in, dim, bias=False) if dim_in is not None else nn.Identity()
self.project_out = nn.Linear(dim, dim_out, bias=False) if dim_out is not None else nn.Identity()
hidden_dim = dim
num_heads = dim_heads
mlp_ratio = 4.0
self._latent_seq_len = _latent_seq_len
self.layers = nn.ModuleList(
[
AudioXMMDiTBlock(
hidden_dim,
num_heads,
mlp_ratio=mlp_ratio,
prefix=f"layers.{i}",
)
for i in range(depth)
]
)
self.proj_mm_tokens = nn.Linear(768, hidden_dim) if dim != 768 else nn.Identity()
self.proj_mm_seq_len = nn.Linear(384, self._latent_seq_len) if self._latent_seq_len != 384 else nn.Identity()
# AudioX RoPE: interleaved (GPT-J) pair layout, theta=10000.
head_dim = hidden_dim // num_heads
pos = torch.arange(self._latent_seq_len, dtype=torch.float32, device=self.device)
inv_freq = 1.0 / (10000 ** (torch.arange(0, head_dim, 2, dtype=torch.float32, device=self.device) / head_dim))
ang = torch.outer(pos, inv_freq)
self.register_buffer("latent_rope_cos", torch.cos(ang), persistent=False)
self.register_buffer("latent_rope_sin", torch.sin(ang), persistent=False)
@property
def device(self):
return next(self.parameters()).device
def forward(
self,
x,
prepend_embeds=None,
context=None,
):
x = self.project_in(x)
if prepend_embeds is not None:
prepend_dim = prepend_embeds.shape[-1]
assert prepend_dim == x.shape[-1], "prepend dimension must match sequence dimension"
x = torch.cat((prepend_embeds, x), dim=-2)
time_cond = prepend_embeds.squeeze(1)
mm_tokens = context
mm_tokens = self.proj_mm_tokens(mm_tokens)
mm_tokens = rearrange(mm_tokens, "b s d -> b d s")
mm_tokens = self.proj_mm_seq_len(mm_tokens)
mm_tokens = rearrange(mm_tokens, "b d s -> b s d")
time_cond = time_cond.unsqueeze(1)
rot = (
self.latent_rope_cos.to(device=x.device, dtype=x.dtype),
self.latent_rope_sin.to(device=x.device, dtype=x.dtype),
)
for block in self.layers:
x = block(x, mm_tokens, rot, context=time_cond)
x = self.project_out(x)
return x
def __getattr__(name: str):
if name in __all__:
return globals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,922 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import json
import math
import os
from collections.abc import Iterable
from typing import Any, ClassVar
import torch
import torch.nn.functional as F
import torchsde
from diffusers import AutoencoderOobleck
from einops import rearrange
from torch import einsum, nn
from torchvision import transforms
from transformers import AutoConfig, CLIPVisionModelWithProjection, T5EncoderModel, T5TokenizerFast
from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.audiox.audiox_transformer import MMDiffusionTransformer
from vllm_omni.diffusion.models.interface import SupportAudioOutput
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.transformers_utils.processors import audiox as _audiox_transforms
_VIDEO_ONLY_TASKS = _audiox_transforms.VIDEO_ONLY_TASKS
_TEXT_VIDEO_TASKS = _audiox_transforms.TEXT_VIDEO_TASKS
_VIDEO_CONDITIONED_TASKS = _audiox_transforms.VIDEO_CONDITIONED_TASKS
_normalize_prompts = _audiox_transforms.normalize_prompts
prepare_audio_reference = _audiox_transforms.prepare_audio_reference
prepare_video_reference = _audiox_transforms.prepare_video_reference
# Polyexponential sigma schedule defaults; mirror upstream AudioX gradio interface
# (``audiox/interface/gradio.py`` ``generate_cond`` defaults: ``sigma_min=0.03``, ``sigma_max=1000``).
_DEFAULT_UPSTREAM_SIGMA_MIN = 0.03
_DEFAULT_UPSTREAM_SIGMA_MAX = 1000.0
logger = init_logger(__name__)
def _load_audiox_bundle_config(model_root: str) -> dict[str, Any]:
with open(os.path.join(os.path.abspath(model_root), "config.json"), encoding="utf-8") as f:
return json.load(f)
def _audio_conditioning_input_samples(model_config: dict[str, Any]) -> int:
configs = model_config["model"]["conditioning"]["configs"]
cfg = next(c["config"] for c in configs if c["id"] == "audio_prompt")
return int(cfg["latent_seq_len"]) * int(cfg["pretransform_config"]["config"]["downsampling_ratio"])
def get_audiox_post_process_func(od_config: OmniDiffusionConfig):
"""Convert the pipeline's float audio tensor to a CPU numpy array for serving."""
def post_process_func(audio: torch.Tensor) -> Any:
if isinstance(audio, torch.Tensor):
return audio.detach().cpu().float().numpy()
return audio
return post_process_func
class SA_PreNorm(nn.Module):
def __init__(self, dim, fn):
super().__init__()
self.norm = nn.LayerNorm(dim)
self.fn = fn
def forward(self, x, **kwargs):
return self.fn(self.norm(x), **kwargs)
class SA_FeedForward(nn.Module):
def __init__(self, dim, hidden_dim):
super().__init__()
# Dropout p=0 preserves upstream ``net.{2,4}`` state-dict keys so the upstream weights
# load into the right slots; inference is identical to no dropout.
self.net = nn.Sequential(
nn.Linear(dim, hidden_dim),
nn.GELU(),
nn.Dropout(0.0),
nn.Linear(hidden_dim, dim),
nn.Dropout(0.0),
)
def forward(self, x):
return self.net(x)
# Manual einsum+softmax only. SDPA/diffusion Attention here degrades conditioning vs upstream.
class SA_Attention(nn.Module):
def __init__(self, dim, heads=8, dim_head=64):
super().__init__()
inner_dim = dim_head * heads
project_out = not (heads == 1 and dim_head == dim)
self.heads = heads
self.scale = dim_head**-0.5
self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False)
self.to_out = (
nn.Sequential(
nn.Linear(inner_dim, dim),
nn.Dropout(0.0),
)
if project_out
else nn.Identity()
)
def forward(self, x):
h = self.heads
qkv = self.to_qkv(x).chunk(3, dim=-1)
q, k, v = (rearrange(t, "b n (h d) -> b h n d", h=h) for t in qkv)
dots = einsum("b h i d, b h j d -> b h i j", q, k) * self.scale
attn = dots.softmax(dim=-1)
out = einsum("b h i j, b h j d -> b h i d", attn, v)
out = rearrange(out, "b h n d -> b n (h d)")
out = self.to_out(out)
return out
class SA_Transformer(nn.Module):
def __init__(self, dim, depth, heads, dim_head, mlp_dim):
super().__init__()
self.layers = nn.ModuleList([])
self.norm = nn.LayerNorm(dim)
for _ in range(depth):
self.layers.append(
nn.ModuleList(
[
SA_PreNorm(dim, SA_Attention(dim, heads=heads, dim_head=dim_head)),
SA_PreNorm(dim, SA_FeedForward(dim, mlp_dim)),
]
)
)
def forward(self, x):
for attn, ff in self.layers:
x = attn(x) + x
x = ff(x) + x
return self.norm(x)
_AUDIOX_OOBLECK_CONFIG = {
"audio_channels": 2,
"channel_multiples": [1, 2, 4, 8, 16],
"decoder_channels": 128,
"decoder_input_channels": 64,
"downsampling_ratios": [2, 4, 4, 8, 8],
"encoder_hidden_size": 128,
"sampling_rate": 44100,
}
def _build_audiox_oobleck(scaling_factor: float = 1.0) -> AutoencoderOobleck:
vae = AutoencoderOobleck(**_AUDIOX_OOBLECK_CONFIG)
vae.audiox_scaling_factor = float(scaling_factor) # type: ignore[attr-defined]
return vae.eval().requires_grad_(False)
class AudioVaePromptAdapter(nn.Module):
def __init__(self, *, cond_dim: int, latent_seq_len: int = 215):
super().__init__()
self.pretransform = _build_audiox_oobleck()
in_ch = int(self.pretransform.config.decoder_input_channels)
self.proj_features_128 = nn.Linear(latent_seq_len, 128)
self.proj_out = nn.Linear(in_ch, cond_dim) if in_ch != cond_dim else nn.Identity()
def forward(self, audio: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
z = self.pretransform.encode(audio, return_dict=True).latent_dist.sample()
latents = z / float(self.pretransform.audiox_scaling_factor)
latents = rearrange(self.proj_features_128(latents), "b c s -> b s c")
latents = self.proj_out(latents)
ones = torch.ones(latents.shape[0], latents.shape[2], device=latents.device)
return latents, ones
class _MAFCrossAttentionBlock(nn.Module):
def __init__(self, dim: int, num_heads: int):
super().__init__()
assert dim % num_heads == 0, "dim must be divisible by num_heads"
self._num_heads = num_heads
self._head_dim = dim // num_heads
self._scale = self._head_dim**-0.5
self.to_q = nn.Linear(dim, dim, bias=True)
self.to_kv = nn.Linear(dim, dim * 2, bias=True)
self.to_out = nn.Linear(dim, dim, bias=True)
nn.init.zeros_(self.to_out.weight)
if self.to_out.bias is not None:
nn.init.zeros_(self.to_out.bias)
def forward(self, experts: torch.Tensor, full_context: torch.Tensor) -> torch.Tensor:
nh, hd = self._num_heads, self._head_dim
q = rearrange(self.to_q(experts), "b n (h d) -> b h n d", h=nh, d=hd)
k, v = (rearrange(t, "b n (h d) -> b h n d", h=nh, d=hd) for t in self.to_kv(full_context).chunk(2, dim=-1))
out = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0, is_causal=False, scale=self._scale)
return self.to_out(rearrange(out, "b h n d -> b n (h d)"))
class _MAFFusionBlock(nn.Module):
def __init__(self, dim: int, num_heads: int, mlp_ratio: float):
super().__init__()
assert dim % num_heads == 0, "dim must be divisible by num_heads"
head_dim = dim // num_heads
self._num_heads = num_heads
hidden = int(dim * mlp_ratio)
self.norm1 = nn.LayerNorm(dim)
self.to_qkv = nn.Linear(dim, dim * 3, bias=True)
self.self_attn = Attention(
num_heads=num_heads,
head_size=head_dim,
softmax_scale=head_dim**-0.5,
causal=False,
)
self.out_proj = nn.Linear(dim, dim, bias=True)
self.norm2 = nn.LayerNorm(dim)
self.ff = nn.Sequential(
nn.Linear(dim, hidden),
nn.GELU(),
nn.Linear(hidden, dim),
)
nn.init.zeros_(self.out_proj.weight)
if self.out_proj.bias is not None:
nn.init.zeros_(self.out_proj.bias)
def forward(self, x: torch.Tensor) -> torch.Tensor:
h = self.norm1(x)
q, k, v = self.to_qkv(h).chunk(3, dim=-1)
nh = self._num_heads
q_bsn = rearrange(q, "b n (h d) -> b n h d", h=nh).contiguous()
k_bsn = rearrange(k, "b n (h d) -> b n h d", h=nh).contiguous()
v_bsn = rearrange(v, "b n (h d) -> b n h d", h=nh).contiguous()
out = self.self_attn(q_bsn, k_bsn, v_bsn, attn_metadata=None)
out = rearrange(out, "b n h d -> b n (h d)")
out = self.out_proj(out)
x = x + out
x = x + self.ff(self.norm2(x))
return x
class MAF_Block(nn.Module):
DIM = 768
MLP_RATIO = 4.0
def __init__(
self,
*,
dim: int = 768,
num_experts_per_modality: int = 64,
num_heads: int = 24,
num_fusion_layers: int = 8,
mlp_ratio: float = 4.0,
):
super().__init__()
total_experts = num_experts_per_modality * 3
self.gating_network = nn.Sequential(
nn.Linear(dim * 3, dim),
nn.GELU(),
nn.Linear(dim, 3),
nn.Sigmoid(),
)
self.unified_experts = nn.Parameter(torch.randn(total_experts, dim))
self.cross_block = _MAFCrossAttentionBlock(dim, num_heads)
self.norm1 = nn.LayerNorm(dim)
self.fusion_blocks = nn.ModuleList(
[_MAFFusionBlock(dim, num_heads, mlp_ratio) for _ in range(num_fusion_layers)]
)
self.norm_v2 = nn.LayerNorm(dim)
self.norm_t2 = nn.LayerNorm(dim)
self.norm_a2 = nn.LayerNorm(dim)
self.bypass_gate_v = nn.Parameter(torch.tensor(-10.0))
self.bypass_gate_t = nn.Parameter(torch.tensor(-10.0))
self.bypass_gate_a = nn.Parameter(torch.tensor(-10.0))
def forward(
self,
video_tokens: torch.Tensor,
text_tokens: torch.Tensor,
audio_tokens: torch.Tensor,
) -> dict[str, torch.Tensor]:
batch_size = video_tokens.shape[0]
v_global = video_tokens.mean(dim=1)
t_global = text_tokens.mean(dim=1)
a_global = audio_tokens.mean(dim=1)
all_global = torch.cat([v_global, t_global, a_global], dim=1)
gates = self.gating_network(all_global)
w_v, w_t, w_a = gates.chunk(3, dim=-1)
gated_v = video_tokens * w_v.unsqueeze(-1)
gated_t = text_tokens * w_t.unsqueeze(-1)
gated_a = audio_tokens * w_a.unsqueeze(-1)
full_context = torch.cat([gated_v, gated_t, gated_a], dim=1)
experts = self.unified_experts.unsqueeze(0).expand(batch_size, -1, -1)
cross_out = self.cross_block(experts, full_context)
updated_experts = self.norm1(experts + cross_out)
for blk in self.fusion_blocks:
updated_experts = blk(updated_experts)
fused_v_experts, fused_t_experts, fused_a_experts = updated_experts.chunk(3, dim=1)
refinement_v = fused_v_experts.mean(dim=1)
refinement_t = fused_t_experts.mean(dim=1)
refinement_a = fused_a_experts.mean(dim=1)
alpha_v = torch.sigmoid(self.bypass_gate_v)
alpha_t = torch.sigmoid(self.bypass_gate_t)
alpha_a = torch.sigmoid(self.bypass_gate_a)
final_v = video_tokens + alpha_v * self.norm_v2(refinement_v).unsqueeze(1)
final_t = text_tokens + alpha_t * self.norm_t2(refinement_t).unsqueeze(1)
final_a = audio_tokens + alpha_a * self.norm_a2(refinement_a).unsqueeze(1)
return {
"video": final_v,
"text": final_t,
"audio": final_a,
}
class _BrownianTreeNoiseSampler:
"""Brownian-tree noise sampler for DPM-Solver++ SDE, ported from k-diffusion.
Returns a scaled Brownian increment between two sigma levels; the tree is indexed by
transformed ``sigma`` (here linear, same as k-diffusion's default), so re-querying
identical (sigma, sigma_next) pairs returns the same noise. Entropy must come from
the user's seed — otherwise the unseeded global RNG would make sampling non-deterministic.
"""
def __init__(self, x: torch.Tensor, sigma_min: torch.Tensor, sigma_max: torch.Tensor, entropy: int):
t0 = torch.as_tensor(sigma_min)
t1 = torch.as_tensor(sigma_max)
self._tree = torchsde.BrownianTree(t0, torch.zeros_like(x), t1, entropy=entropy)
def __call__(self, sigma: torch.Tensor, sigma_next: torch.Tensor) -> torch.Tensor:
# AudioX denoises from sigma_max → 0, so sigma > sigma_next throughout the loop.
t0 = torch.as_tensor(sigma_next)
t1 = torch.as_tensor(sigma)
return -self._tree(t0, t1) / (t1 - t0).sqrt()
class AudioXPipeline(nn.Module, SupportAudioOutput, DiffusionPipelineProfilerMixin):
support_audio_output: ClassVar[bool] = True
audio_sample_rate: ClassVar[int] = 44100
audio_channels: ClassVar[int] = 2
_PROFILER_TARGETS: ClassVar[list[str]] = ["diffuse"]
_CLIP_SYNC_DURATION_SEC: ClassVar[float] = 10.0
_VIDEO_SYNC_FRAME_COUNT: ClassVar[int] = 240
def __init__(
self,
*,
od_config: OmniDiffusionConfig,
prefix: str = "",
):
super().__init__()
self.od_config = od_config
self.device = get_local_device()
if od_config.model is None:
raise ValueError(
"AudioXPipeline requires od_config.model (directory with unified safetensors; "
"see https://huggingface.co/zhangj1an/AudioX)."
)
if os.path.exists(od_config.model):
self._model_root = os.path.abspath(od_config.model)
else:
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
self._model_root = download_weights_from_hf_specific(od_config.model, None, ["*"])
self._model_config = _load_audiox_bundle_config(self._model_root)
model_config = self._model_config["model"]
diffusion_config = model_config["diffusion"]
self.model = MMDiffusionTransformer(**dict(diffusion_config["config"]))
cond_configs = {c["id"]: c["config"] for c in model_config["conditioning"]["configs"]}
self.audio_vae_adapter = AudioVaePromptAdapter(
cond_dim=int(model_config["conditioning"]["cond_dim"]),
latent_seq_len=int(cond_configs["audio_prompt"]["latent_seq_len"]),
)
t5_name = cond_configs["text_prompt"]["t5_model_name"]
self._t5_max_length = int(cond_configs["text_prompt"]["max_length"])
self.tokenizer = T5TokenizerFast.from_pretrained(t5_name)
t5_config = AutoConfig.from_pretrained(t5_name)
self.text_encoder = T5EncoderModel(t5_config).train(False).requires_grad_(False).to(torch.float16)
clip_name = cond_configs["video_prompt"]["clip_model_name"]
clip_config = AutoConfig.from_pretrained(clip_name)
self.clip_encoder = CLIPVisionModelWithProjection(clip_config.vision_config)
_CLIP_PATCH_TOKENS, _VIDEO_FPS, _DURATION_SEC, _DIM = 50, 5, 10, 768
_in_features = _CLIP_PATCH_TOKENS * _VIDEO_FPS * _DURATION_SEC
self._clip_in_features = _in_features
self._clip_out_features = 128
self.clip_proj = nn.Linear(_in_features, self._clip_out_features)
self.clip_proj_sync = nn.Linear(240, self._clip_out_features)
self.clip_sync_weight = nn.Parameter(torch.tensor(0.0))
self.clip_temp_transformer = SA_Transformer(_DIM, depth=4, heads=16, dim_head=64, mlp_dim=_DIM * 4)
self.clip_temp_pos_embedding = nn.Parameter(torch.randn(1, _VIDEO_FPS * _DURATION_SEC, _DIM))
self.clip_empty_visual_feat = nn.Parameter(torch.zeros(1, self._clip_out_features, _DIM), requires_grad=False)
_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
_CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
self._clip_normalize = transforms.Compose([transforms.Normalize(mean=list(_CLIP_MEAN), std=list(_CLIP_STD))])
self.pretransform = _build_audiox_oobleck(
scaling_factor=float(model_config["pretransform"].get("scale", 1.0)),
)
self.io_channels = model_config["io_channels"]
self.diffusion_objective = "v"
gate_type_config = diffusion_config["gate_type_config"]
self.maf_block = MAF_Block(
dim=768,
num_experts_per_modality=int(gate_type_config["num_experts_per_modality"]),
num_heads=int(gate_type_config["num_heads"]),
num_fusion_layers=int(gate_type_config["num_fusion_layers"]),
)
logger.debug("AudioX model built from %s", self._model_root)
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=self._model_root,
subfolder="transformer",
revision=getattr(od_config, "revision", None),
prefix="",
),
]
sample_rate = int(self._model_config.get("sample_rate", 48000))
self._sample_rate = sample_rate
self._sample_size = int(self._model_config.get("sample_size", sample_rate * 10))
self._target_fps = int(self._model_config.get("video_fps", 5))
self._audio_conditioning_samples = _audio_conditioning_input_samples(self._model_config)
self.setup_diffusion_pipeline_profiler(
profiler_targets=list(self._PROFILER_TARGETS),
enable_diffusion_pipeline_profiler=od_config.enable_diffusion_pipeline_profiler,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
_legacy_prefix = "conditioner.conditioners.audio_prompt."
# DiT self-attn QKV: bundle stores weights with last dim (h, d, qkv) interleaved
# (i.e. Q/K/V rows for the same head sit next to each other), whereas
# QKVParallelLinear expects stacked [Q|K|V] blocks along the output axis.
# Reshape once at load time so the parallel loader can consume them.
nheads = int(self._model_config["model"]["diffusion"]["config"]["num_heads"])
embed_dim = int(self._model_config["model"]["diffusion"]["config"]["embed_dim"])
total_heads = embed_dim // nheads
head_dim = embed_dim // total_heads
qkv_mid = ".attn.qkv."
to_kv_mid = ".cross_attn.to_kv."
def _restack_interleaved(tensor: torch.Tensor, n_slots: int) -> torch.Tensor:
"""Turn bundle's (h, d, slot) interleaved-last-dim layout into stacked [slot|slot|...]."""
if tensor.dim() == 2: # weight: (out, in)
out, inp = tensor.shape
assert out == n_slots * total_heads * head_dim, (out, n_slots, total_heads, head_dim)
return (
tensor.view(total_heads, head_dim, n_slots, inp).permute(2, 0, 1, 3).reshape(out, inp).contiguous()
)
out = tensor.shape[0]
assert out == n_slots * total_heads * head_dim
return tensor.view(total_heads, head_dim, n_slots).permute(2, 0, 1).reshape(out).contiguous()
def _remap(items):
for name, tensor in items:
if name.startswith(_legacy_prefix):
name = "audio_vae_adapter." + name[len(_legacy_prefix) :]
if qkv_mid in name and (name.endswith(".weight") or name.endswith(".bias")):
tensor = _restack_interleaved(tensor, 3)
elif to_kv_mid in name and (name.endswith(".weight") or name.endswith(".bias")):
tensor = _restack_interleaved(tensor, 2)
yield name, tensor
loaded = AutoWeightsLoader(self).load_weights(_remap(weights))
self.to(torch.float32)
self.eval().requires_grad_(False)
return loaded
def _conditioning_dtype(self) -> torch.dtype:
p = next(self.model.parameters())
return p.dtype if p.dtype.is_floating_point else torch.float32
@staticmethod
def _normalize_task(task: str | None) -> str | None:
if task is None:
return None
t = str(task).strip().lower()
return t or None
@staticmethod
def _text_for_task(task_norm: str | None, prompt: str) -> str:
if task_norm in _VIDEO_ONLY_TASKS:
return ""
return prompt
@staticmethod
def _ensure_text_video_prompts(task_norm: str | None, prompts: list[str]) -> None:
if task_norm not in _TEXT_VIDEO_TASKS:
return
for i, p in enumerate(prompts):
if not str(p).strip():
raise ValueError(
f"audiox_task={task_norm!r} requires a non-empty text prompt for item {i}; "
"use v2a/v2m for video-only generation."
)
def _audio_prompt_tensors(
self,
*,
raw_prompts: list[Any],
extra: dict[str, Any],
device: torch.device,
cond_dtype: torch.dtype = torch.float32,
) -> list[torch.Tensor]:
target_len = self._audio_conditioning_samples
sample_rate = self._sample_rate
seconds_start = float(extra.get("seconds_start", 0.0))
seconds_total = float(target_len) / float(sample_rate)
out: list[torch.Tensor] = []
for raw in raw_prompts:
src = extra.get("audio_path")
if src is None:
out.append(torch.zeros(2, target_len, device=device, dtype=cond_dtype))
continue
wav = prepare_audio_reference(
src,
model_sample_rate=sample_rate,
seconds_start=seconds_start,
seconds_total=seconds_total,
device=device,
)
out.append(wav.to(dtype=cond_dtype))
return out
def _video_feature_tensors(
self,
*,
task_norm: str | None,
raw_prompts: list[Any],
extra: dict[str, Any],
seconds_start: float,
target_fps: int,
device: torch.device,
cond_dtype: torch.dtype = torch.float32,
) -> list[torch.Tensor]:
clip_frames = int(round(self._CLIP_SYNC_DURATION_SEC * target_fps))
if task_norm not in _VIDEO_CONDITIONED_TASKS:
empty = torch.zeros(clip_frames, 3, 224, 224, device=device, dtype=cond_dtype)
return [empty for _ in raw_prompts]
tensors: list[torch.Tensor] = []
for _ in raw_prompts:
src = extra.get("video_path")
if src is None:
raise ValueError(f"audiox_task={task_norm!r} requires video input: set extra_args['video_path'].")
vt = prepare_video_reference(
src,
duration=float(self._CLIP_SYNC_DURATION_SEC),
target_fps=target_fps,
seek_time=seconds_start,
)
tensors.append(vt.to(device=device, dtype=cond_dtype))
return tensors
def get_conditioning_inputs(self, conditioning_tensors: dict[str, Any], negative: bool = False) -> dict[str, Any]:
video_feature, video_mask = conditioning_tensors["video_prompt"]
text_feature, text_mask = conditioning_tensors["text_prompt"]
audio_feature, audio_mask = conditioning_tensors["audio_prompt"]
refined = self.maf_block(text_feature, video_feature, audio_feature)
fused = torch.cat(list(refined.values()), dim=1)
masks = torch.cat([video_mask, text_mask, audio_mask], dim=1)
if negative:
return {"negative_cross_attn_cond": fused, "negative_cross_attn_mask": masks}
return {"cross_attn_cond": fused}
def diffuse(
self,
*,
steps: int,
guidance_scale: float,
conditioning_tensors: dict[str, Any],
negative_conditioning_tensors: dict[str, Any] | None,
batch_size: int,
sigma_min: float,
sigma_max: float,
generator: torch.Generator,
cfg_rescale: float,
) -> torch.Tensor:
device = self.device
model_dtype = next(self.model.parameters()).dtype
# Match upstream AudioX: disable TF32 matmul + fp16 reduced precision + cudnn benchmark
# for numerical parity with audiox/inference/generation.py:152-156.
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
torch.backends.cudnn.benchmark = False
latent_len = self._sample_size // int(self.pretransform.hop_length)
noise = torch.randn(
[batch_size, self.io_channels, latent_len], device=device, generator=generator, dtype=model_dtype
)
def _cast(d: dict[str, Any]) -> dict[str, Any]:
return {k: (v.type(model_dtype) if isinstance(v, torch.Tensor) else v) for k, v in d.items()}
cond = _cast(self.get_conditioning_inputs(conditioning_tensors))
neg = (
_cast(self.get_conditioning_inputs(negative_conditioning_tensors, negative=True))
if negative_conditioning_tensors is not None
else {}
)
# Inlined k-diffusion VDenoiser + sample_dpmpp_3m_sde, matching upstream AudioX exactly.
# diffusers' EDMDPMSolverMultistepScheduler uses different v-prediction preconditioning
# and a different stochastic update rule, which here produces a fixed ~861 Hz resonance
# in the decoded audio regardless of conditioning.
def denoise(x_in: torch.Tensor, sigma: torch.Tensor) -> torch.Tensor:
s2 = sigma * sigma
c_skip = 1.0 / (s2 + 1.0)
c_out = -sigma / (s2 + 1.0).sqrt()
c_in = 1.0 / (s2 + 1.0).sqrt()
t_cond = sigma.atan() * (2.0 / math.pi)
v = self.model(
x_in * c_in,
t_cond,
cross_attn_cond=cond["cross_attn_cond"],
negative_cross_attn_cond=neg.get("negative_cross_attn_cond"),
negative_cross_attn_mask=neg.get("negative_cross_attn_mask"),
cfg_scale=guidance_scale,
scale_phi=cfg_rescale,
)
return v * c_out + x_in * c_skip
ramp = torch.linspace(1.0, 0.0, steps, device=device)
sigmas = torch.cat(
[
torch.exp(ramp * (math.log(sigma_max) - math.log(sigma_min)) + math.log(sigma_min)),
torch.zeros(1, device=device),
]
)
x = noise * sigmas[0]
# Match upstream AudioX: sampler runs under fp16 autocast (see audiox/inference/sampling.py:184).
with torch.cuda.amp.autocast():
if steps <= 1:
s_in = x.new_ones([x.shape[0]])
sampled = denoise(x, sigmas[0] * s_in)
else:
# DPM-Solver++(3M) SDE loop (k-diffusion sample_dpmpp_3m_sde), eta=1.0, s_noise=1.0.
noise_sampler = _BrownianTreeNoiseSampler(
x,
sigmas[sigmas > 0].min(),
sigmas.max(),
entropy=generator.initial_seed(),
)
s_in = x.new_ones([x.shape[0]])
denoised_1 = denoised_2 = None
h_1 = h_2 = None
eta = 1.0
for i in range(len(sigmas) - 1):
denoised = denoise(x, sigmas[i] * s_in)
if sigmas[i + 1] == 0:
x = denoised
else:
t_, s_ = -sigmas[i].log(), -sigmas[i + 1].log()
h = s_ - t_
h_eta = h * (eta + 1)
x = torch.exp(-h_eta) * x + (-h_eta).expm1().neg() * denoised
if h_2 is not None:
r0 = h_1 / h
r1 = h_2 / h
d1_0 = (denoised - denoised_1) / r0
d1_1 = (denoised_1 - denoised_2) / r1
d1 = d1_0 + (d1_0 - d1_1) * r0 / (r0 + r1)
d2 = (d1_0 - d1_1) / (r0 + r1)
phi_2 = h_eta.neg().expm1() / h_eta + 1
phi_3 = phi_2 / h_eta - 0.5
x = x + phi_2 * d1 - phi_3 * d2
elif h_1 is not None:
r = h_1 / h
d = (denoised - denoised_1) / r
phi_2 = h_eta.neg().expm1() / h_eta + 1
x = x + phi_2 * d
x = (
x
+ noise_sampler(sigmas[i], sigmas[i + 1])
* sigmas[i + 1]
* (-2 * h * eta).expm1().neg().sqrt()
)
denoised_1, denoised_2 = denoised, denoised_1
h_1, h_2 = h, h_1
sampled = x
vae = self.pretransform.to(device=sampled.device, dtype=torch.float32).eval()
return vae.decode(sampled.to(torch.float32) * float(vae.audiox_scaling_factor), return_dict=True).sample
def _encode_text(self, texts: list[str], device: torch.device) -> list[torch.Tensor]:
self.text_encoder.to(device)
encoded = self.tokenizer(
texts,
truncation=True,
max_length=self._t5_max_length,
padding="max_length",
return_tensors="pt",
)
input_ids = encoded["input_ids"].to(device)
attention_mask = encoded["attention_mask"].to(device).to(torch.bool)
self.text_encoder.eval()
with torch.no_grad():
embeddings = self.text_encoder(input_ids=input_ids, attention_mask=attention_mask)["last_hidden_state"]
embeddings = embeddings.float() * attention_mask.unsqueeze(-1).float()
return [embeddings, attention_mask]
def _encode_video(self, video_list: list[dict], device: torch.device) -> list[torch.Tensor]:
self.clip_encoder.to(device).eval()
video_tensors = [item["video_tensors"] for item in video_list]
video_sync_frames = torch.cat([item["video_sync_frames"] for item in video_list], dim=0).to(device)
original_videos = torch.cat(video_tensors, dim=0).to(device)
batch_size, time_length, _, _, _ = original_videos.size()
is_zero = torch.all(original_videos == 0, dim=(1, 2, 3, 4))
frames = original_videos.flatten(0, 1)
pixel_values = self._clip_normalize(frames).to(device)
with torch.no_grad():
outputs = self.clip_encoder(pixel_values=pixel_values)
hidden = outputs.last_hidden_state
hidden = rearrange(hidden, "(b t) p d -> (b p) t d", b=batch_size, t=time_length)
hidden = hidden + self.clip_temp_pos_embedding
hidden = self.clip_temp_transformer(hidden)
hidden = rearrange(hidden, "(b p) t d -> b (t p) d", b=batch_size)
hidden = self.clip_proj(hidden.view(-1, self._clip_in_features))
hidden = hidden.view(batch_size, self._clip_out_features, -1)
sync = self.clip_proj_sync(video_sync_frames.view(-1, 240))
sync = sync.view(batch_size, self._clip_out_features, -1)
hidden = hidden + self.clip_sync_weight * sync
empty = self.clip_empty_visual_feat.expand(batch_size, -1, -1)
hidden = torch.where(is_zero.view(batch_size, 1, 1), empty, hidden)
return [hidden, torch.ones(batch_size, 1, device=device)]
def _encode_conditioning_tensors(self, batch_metadata: list[dict[str, Any]]) -> dict[str, Any]:
device = self.device
audio = torch.cat([item["audio_prompt"] for item in batch_metadata], dim=0).to(device)
return {
"audio_prompt": list(self.audio_vae_adapter(audio)),
"text_prompt": self._encode_text([item["text_prompt"] for item in batch_metadata], device),
"video_prompt": self._encode_video([item["video_prompt"] for item in batch_metadata], device),
}
def _build_conditioning_batch(
self,
*,
texts: list[str],
video_tensors_list: list[torch.Tensor],
audio_prompt_list: list[torch.Tensor],
sync_features: torch.Tensor,
seconds_start: float,
seconds_model: float,
num_outputs_per_prompt: int,
task_norm: str | None,
) -> list[dict[str, Any]]:
batch: list[dict[str, Any]] = []
for i, text in enumerate(texts):
for _ in range(num_outputs_per_prompt):
batch.append(
{
"video_prompt": {
"video_tensors": video_tensors_list[i].unsqueeze(0),
"video_sync_frames": sync_features,
},
"text_prompt": self._text_for_task(task_norm, text),
"audio_prompt": audio_prompt_list[i].unsqueeze(0),
"seconds_start": seconds_start,
"seconds_total": seconds_model,
}
)
return batch
def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
if req.prompts is None or len(req.prompts) == 0:
raise ValueError("AudioXPipeline requires at least one prompt.")
normalized_prompts = _normalize_prompts(list(req.prompts))
prompts = [p["prompt"] for p in normalized_prompts]
sampling_params = req.sampling_params
if sampling_params.num_inference_steps is None:
raise ValueError("AudioXPipeline requires sampling_params.num_inference_steps.")
num_inference_steps = int(sampling_params.num_inference_steps)
extra_args = sampling_params.extra_args or {}
task_norm = self._normalize_task(extra_args.get("audiox_task"))
self._ensure_text_video_prompts(task_norm, prompts)
neg: list[str] | None = None
if not all(p.get("negative_prompt") is None for p in normalized_prompts):
neg = [str(p.get("negative_prompt") or "") for p in normalized_prompts]
guidance_scale = float(sampling_params.guidance_scale)
if sampling_params.num_outputs_per_prompt <= 0:
raise ValueError("AudioXPipeline requires sampling_params.num_outputs_per_prompt > 0.")
num_outputs_per_prompt = int(sampling_params.num_outputs_per_prompt)
batch_size = len(prompts) * num_outputs_per_prompt
seconds_start = float(extra_args.get("seconds_start", 0.0))
seconds_model = self._sample_size / self._sample_rate
seconds_total = float(extra_args.get("seconds_total", seconds_model))
sigma_min = float(extra_args.get("sigma_min", _DEFAULT_UPSTREAM_SIGMA_MIN))
sigma_max = float(extra_args.get("sigma_max", _DEFAULT_UPSTREAM_SIGMA_MAX))
cfg_rescale = float(extra_args.get("cfg_rescale", 0.0))
device = self.device
generator = sampling_params.generator
if generator is None:
raise ValueError("AudioXPipeline requires sampling_params.generator.")
target_fps = self._target_fps
cond_dtype = self._conditioning_dtype()
sync_features = torch.zeros(1, self._VIDEO_SYNC_FRAME_COUNT, 768, device=device, dtype=cond_dtype)
audio_prompt_list = self._audio_prompt_tensors(
raw_prompts=normalized_prompts,
extra=extra_args,
device=device,
cond_dtype=cond_dtype,
)
video_tensors_list = self._video_feature_tensors(
task_norm=task_norm,
raw_prompts=normalized_prompts,
extra=extra_args,
seconds_start=seconds_start,
target_fps=target_fps,
device=device,
cond_dtype=cond_dtype,
)
conditioning_batch = self._build_conditioning_batch(
texts=prompts,
video_tensors_list=video_tensors_list,
audio_prompt_list=audio_prompt_list,
sync_features=sync_features,
seconds_start=seconds_start,
seconds_model=seconds_model,
num_outputs_per_prompt=num_outputs_per_prompt,
task_norm=task_norm,
)
negative_conditioning_batch: list[dict[str, Any]] | None = None
if neg is not None and guidance_scale > 1.0:
negative_conditioning_batch = self._build_conditioning_batch(
texts=neg,
video_tensors_list=video_tensors_list,
audio_prompt_list=audio_prompt_list,
sync_features=sync_features,
seconds_start=seconds_start,
seconds_model=seconds_model,
num_outputs_per_prompt=num_outputs_per_prompt,
task_norm=task_norm,
)
conditioning_tensors = self._encode_conditioning_tensors(conditioning_batch)
negative_conditioning_tensors: dict[str, Any] | None = None
if negative_conditioning_batch is not None:
negative_conditioning_tensors = self._encode_conditioning_tensors(negative_conditioning_batch)
audio = self.diffuse(
steps=num_inference_steps,
guidance_scale=guidance_scale,
conditioning_tensors=conditioning_tensors,
negative_conditioning_tensors=negative_conditioning_tensors,
batch_size=batch_size,
sigma_min=sigma_min,
sigma_max=sigma_max,
generator=generator,
cfg_rescale=cfg_rescale,
)
# Trim decoded audio to the requested duration (matches upstream AudioX sample script).
if 0.0 < seconds_total < seconds_model:
target_samples = int(seconds_total * self._sample_rate)
audio = audio[..., :target_samples]
return DiffusionOutput(
output=audio,
custom_output={"audiox_task": task_norm},
stage_durations=self.stage_durations
if getattr(self, "enable_diffusion_pipeline_profiler", False)
else None,
)
@@ -5,7 +5,6 @@
Stable Audio DiT Model for vLLM-Omni.
"""
import math
from collections.abc import Iterable
import torch
@@ -18,6 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.distributed.hsdp_utils import is_transformer_block_module
from vllm_omni.diffusion.layers.fourier import GaussianFourierProjection
logger = init_logger(__name__)
@@ -56,7 +56,7 @@ def apply_rotary_emb_stable_audio(
return torch.cat([x_rot, x_pass], dim=-1)
class StableAudioGaussianFourierProjection(nn.Module):
class StableAudioGaussianFourierProjection(GaussianFourierProjection):
"""Gaussian Fourier embeddings for noise levels.
Matches diffusers StableAudioGaussianFourierProjection with:
@@ -65,15 +65,12 @@ class StableAudioGaussianFourierProjection(nn.Module):
"""
def __init__(self, embedding_size: int = 256, scale: float = 1.0):
super().__init__()
self.weight = nn.Parameter(torch.randn(embedding_size) * scale, requires_grad=False)
super().__init__(in_features=1, embedding_size=embedding_size, scale=scale, trainable=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
# x shape: [batch] or [batch, 1]
# Output: [batch, embedding_size * 2]
x_proj = 2 * math.pi * x[:, None] @ self.weight[None, :]
# flip_sin_to_cos=True means cos comes first
return torch.cat([torch.cos(x_proj), torch.sin(x_proj)], dim=-1)
# Output: [batch, embedding_size * 2], with cos first.
return super().forward(x)
class StableAudioSelfAttention(nn.Module):