[Feat][Perf][AuK]Support Online Serving and DiT CUDAGraph (#7469)

Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-30 00:18:38 +00:00
1 parent 3a28882946
commit 35cf6ac355
22 files changed
+1473 -38

No files matched your search

@@ -28,13 +28,13 @@ Deliberate differences from the reference, all inference-only:
"""
import math
from collections.abc import Iterable, Mapping
from collections.abc import Callable, Iterable, Mapping
import torch
import torch.nn.functional as F
from torch import nn
__all__ = ["AuKTransformer", "dit_state_dict", "sample_latents"]
__all__ = ["AuKTransformer", "build_time_grid", "dit_state_dict", "sample_latents"]
def _sdpa(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor:
@@ -86,14 +86,17 @@ class Rotary(nn.Module):
exponents = torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim
return 1.0 / (self.base**exponents)
def forward(self, seq_len: int) -> torch.Tensor:
"""Return frequencies ``[1, 1, seq_len, dim]`` for positions ``0..seq_len-1``."""
def forward(self, seq_len: int, mask: torch.Tensor | None = None) -> torch.Tensor:
"""Return rotary frequencies, compressing positions across padding."""
inv_freq = self.inv_freq
if inv_freq.dtype != torch.float32:
inv_freq = self._frequencies(inv_freq.device)
pos = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32)
freqs = torch.outer(pos, inv_freq)
return torch.stack((freqs, freqs), dim=-1).flatten(-2)[None, None]
if mask is None:
pos = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32)[None]
else:
pos = (mask.to(torch.int32).cumsum(dim=1) - 1).clamp_min(0).to(torch.float32)
freqs = pos.unsqueeze(-1) * inv_freq
return torch.stack((freqs, freqs), dim=-1).flatten(-2).unsqueeze(1)
class TimeEmbedding(nn.Module):
@@ -541,15 +544,15 @@ class AuKTransformer(nn.Module):
seq_len = x.shape[1]
text_len = c.shape[1]
rope_audio = self.rotary_embed(seq_len)
rope_text = self.rotary_embed(text_len)
rope_audio = self.rotary_embed(seq_len, audio_mask)
rope_text = self.rotary_embed(text_len, c_mask)
for block in self.transformer_blocks:
c, x = block(x, c, t, mask=audio_mask, rope=rope_audio, c_rope=rope_text, c_mask=c_mask)
x = torch.cat([c, x], dim=1)
rope = self.rotary_embed(text_len + seq_len)
single_mask = None if audio_mask is None else torch.cat([c_mask, audio_mask], dim=1)
rope = self.rotary_embed(text_len + seq_len, single_mask)
for block in self.single_transformer_blocks:
x = block(x, t, mask=single_mask, rope=rope)
@@ -571,6 +574,31 @@ def dit_state_dict(
return {name[len(prefix) :]: tensor for name, tensor in items if name.startswith(prefix)}
def build_time_grid(
*,
nfe: int,
sway_sampling_coef: float | None,
t_grid: list[float] | None,
device: torch.device | str,
) -> torch.Tensor:
"""Build and validate the Euler schedule before a graph replay."""
if t_grid is not None:
timesteps = torch.tensor(t_grid, device="cpu", dtype=torch.float32)
else:
timesteps = torch.linspace(0, 1, nfe + 1, device="cpu", dtype=torch.float32)
if sway_sampling_coef is not None:
timesteps = timesteps + sway_sampling_coef * (torch.cos(math.pi / 2 * timesteps) - 1 + timesteps)
if (
timesteps.ndim != 1
or timesteps.numel() < 2
or not bool(torch.isfinite(timesteps).all() & torch.all(timesteps[1:] > timesteps[:-1]))
):
raise ValueError(
f"AuK sampling needs a strictly increasing time grid with at least two points; got {timesteps.tolist()}"
)
return timesteps.to(device)
@torch.no_grad()
def sample_latents(
dit: AuKTransformer,
@@ -589,6 +617,7 @@ def sample_latents(
latent_dim: int | None = None,
device: torch.device | str | None = None,
dtype: torch.dtype | None = None,
sampler: Callable[..., torch.Tensor] | None = None,
) -> torch.Tensor:
"""Integrate the flow from noise to audio latents with explicit Euler steps.
@@ -640,25 +669,61 @@ def sample_latents(
.unsqueeze(0)
)
if t_grid is not None:
t = torch.tensor(t_grid, device=device, dtype=torch.float32)
else:
t = torch.linspace(0, 1, nfe + 1, device=device, dtype=torch.float32)
if sway_sampling_coef is not None:
t = t + sway_sampling_coef * (torch.cos(math.pi / 2 * t) - 1 + t)
t = build_time_grid(
nfe=nfe,
sway_sampling_coef=sway_sampling_coef,
t_grid=t_grid,
device=device,
)
if sampler is not None:
try:
for i in range(t.shape[0] - 1):
velocity = sampler(
x=x,
text=text,
c_mask=c_mask,
ref=ref,
ref_mask=ref_mask,
timestep=t[i],
cfg_strength=cfg_strength,
)
x = x + (t[i + 1] - t[i]) * velocity
finally:
dit.clear_cache()
return x
return _sample_latents(
dit,
initial_latents=x,
text=text,
c_mask=c_mask,
ref=ref,
ref_mask=ref_mask,
timesteps=t,
cfg_strength=cfg_strength,
)
def _sample_latents(
dit: AuKTransformer,
*,
initial_latents: torch.Tensor,
text: torch.Tensor,
c_mask: torch.Tensor | None,
ref: torch.Tensor,
ref_mask: torch.Tensor | None,
timesteps: torch.Tensor,
cfg_strength: float,
) -> torch.Tensor:
"""Euler integration shared by eager sampling and CUDA graph capture."""
x = initial_latents
guided = cfg_strength >= 1e-5
if t.numel() < 2 or not bool(torch.all(t[1:] > t[:-1])):
raise ValueError(
f"AuK sampling needs a strictly increasing time grid with at least two points; got {t.tolist()}"
)
try:
for i in range(t.shape[0] - 1):
for i in range(timesteps.shape[0] - 1):
if guided:
pred = dit(
x,
text,
t[i],
timesteps[i],
c_mask=c_mask,
ref=ref,
ref_mask=ref_mask,
@@ -668,8 +733,8 @@ def sample_latents(
v_cond, v_uncond = pred.chunk(2, dim=0)
v = v_cond + (v_cond - v_uncond) * cfg_strength
else:
v = dit(x, text, t[i], c_mask=c_mask, ref=ref, ref_mask=ref_mask)
x = x + (t[i + 1] - t[i]) * v
v = dit(x, text, timesteps[i], c_mask=c_mask, ref=ref, ref_mask=ref_mask)
x = x + (timesteps[i + 1] - timesteps[i]) * v
finally:
# The cached text projections belong to this request only; a failed
# step must not leak them into the next one.
@@ -0,0 +1,242 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Per-shape CUDA graph for one AuK DiT denoise step."""
from __future__ import annotations
from collections import OrderedDict
from dataclasses import dataclass
import torch
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm.utils.math_utils import round_up
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer
logger = init_logger(__name__)
@dataclass
class _GraphEntry:
graph: torch.cuda.CUDAGraph
static_x: torch.Tensor
static_x_mask: torch.Tensor
static_text: torch.Tensor
static_c_mask: torch.Tensor
static_ref: torch.Tensor
static_ref_mask: torch.Tensor
static_timestep: torch.Tensor
static_cfg: torch.Tensor | None
static_out: torch.Tensor
class AuKCUDAGraphWrapper:
"""Replay one DiT denoise step and leave Euler scheduling to the caller.
The graph is keyed by the target, text and reference sequence lengths plus
whether the CFG branch is enabled. Timestep and the CFG strength are
mutable scalar buffers, so all Euler steps and CFG values within one path
reuse one graph.
"""
_TARGET_ALIGNMENT = 64
_TEXT_ALIGNMENT = 64
_REF_ALIGNMENT = 50
def __init__(self, dit: AuKTransformer, *, enabled: bool = True, max_graphs: int = 32) -> None:
self.dit = dit
self.enabled = bool(enabled)
self.max_graphs = max(1, int(max_graphs))
self._cache: OrderedDict[tuple, _GraphEntry] = OrderedDict()
self._pool_handle: int | None = None
logger.info("Initialized AuK DiT lazy CUDA graph cache: enabled=%s slots=%d", self.enabled, self.max_graphs)
@classmethod
def _bucket_inputs(
cls,
x: torch.Tensor,
text: torch.Tensor,
c_mask: torch.Tensor,
ref: torch.Tensor,
ref_mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
target_bucket = round_up(x.shape[1], cls._TARGET_ALIGNMENT)
text_bucket = round_up(text.shape[1], cls._TEXT_ALIGNMENT)
ref_bucket = round_up(ref.shape[1], cls._REF_ALIGNMENT)
x_mask = torch.ones(x.shape[:2], dtype=torch.bool, device=x.device)
return (
F.pad(x, (0, 0, 0, target_bucket - x.shape[1])),
F.pad(x_mask, (0, target_bucket - x_mask.shape[1]), value=False),
F.pad(text, (0, 0, 0, text_bucket - text.shape[1])),
F.pad(c_mask, (0, text_bucket - c_mask.shape[1]), value=False),
F.pad(ref, (0, 0, 0, ref_bucket - ref.shape[1])),
F.pad(ref_mask, (0, ref_bucket - ref_mask.shape[1]), value=False),
)
@staticmethod
def _key(
x: torch.Tensor,
text: torch.Tensor,
ref: torch.Tensor,
uses_cfg: bool,
) -> tuple[int, int, int, bool]:
return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg)
def _retire_graph_generation_if_full(self) -> None:
"""Retire all graphs together so none outlive shared workspaces."""
if len(self._cache) >= self.max_graphs:
self._cache.clear()
@torch.no_grad()
def __call__(
self,
*,
x: torch.Tensor,
text: torch.Tensor,
c_mask: torch.Tensor,
ref: torch.Tensor,
ref_mask: torch.Tensor,
timestep: torch.Tensor,
cfg_strength: float,
) -> torch.Tensor:
uses_cfg = cfg_strength >= 1e-5
if not self.enabled or x.device.type != "cuda" or torch.cuda.is_current_stream_capturing():
if uses_cfg:
return self._run_cfg(x, None, text, c_mask, ref, ref_mask, timestep, cfg_strength=cfg_strength)
return self._run(x, None, text, c_mask, ref, ref_mask, timestep)
target_frames = x.shape[1]
x, x_mask, text, c_mask, ref, ref_mask = self._bucket_inputs(x, text, c_mask, ref, ref_mask)
inputs = (x, x_mask, text, c_mask, ref, ref_mask, timestep)
key = self._key(x, text, ref, uses_cfg)
entry = self._cache.get(key)
if entry is None:
# Same Retirement as #6587, details discussed in #7469
self._retire_graph_generation_if_full()
try:
entry = self._capture(*inputs, cfg_strength=cfg_strength, uses_cfg=uses_cfg)
except Exception:
self._cache.clear()
self.enabled = False
logger.exception(
"Disabling AuK DiT CUDA graphs after capture failure for key=%s; "
"subsequent requests will use eager execution.",
key,
)
raise
self._cache[key] = entry
else:
self._cache.move_to_end(key)
entry.static_x.copy_(x)
entry.static_x_mask.copy_(x_mask)
entry.static_text.copy_(text)
entry.static_c_mask.copy_(c_mask)
entry.static_ref.copy_(ref)
entry.static_ref_mask.copy_(ref_mask)
entry.static_timestep.copy_(timestep)
if entry.static_cfg is not None:
entry.static_cfg.fill_(cfg_strength)
entry.graph.replay()
return entry.static_out[:, :target_frames].clone()
def _run(
self,
x: torch.Tensor,
x_mask: torch.Tensor | None,
text: torch.Tensor,
c_mask: torch.Tensor,
ref: torch.Tensor,
ref_mask: torch.Tensor,
timestep: torch.Tensor,
) -> torch.Tensor:
return self.dit(x, text, timestep, mask=x_mask, c_mask=c_mask, ref=ref, ref_mask=ref_mask)
def _run_cfg(
self,
x: torch.Tensor,
x_mask: torch.Tensor | None,
text: torch.Tensor,
c_mask: torch.Tensor,
ref: torch.Tensor,
ref_mask: torch.Tensor,
timestep: torch.Tensor,
*,
cfg_strength: torch.Tensor | float,
) -> torch.Tensor:
pred = self.dit(
x,
text,
timestep,
mask=x_mask,
c_mask=c_mask,
ref=ref,
ref_mask=ref_mask,
cfg_infer=True,
cache=True,
)
conditional, unconditional = pred.chunk(2, dim=0)
return conditional + (conditional - unconditional) * cfg_strength
def _capture(
self,
*inputs: torch.Tensor,
cfg_strength: float,
uses_cfg: bool,
) -> _GraphEntry:
static_inputs = tuple(value.clone() for value in inputs)
static_cfg = None
if uses_cfg:
static_cfg = torch.empty((), device=static_inputs[0].device, dtype=torch.float32)
static_cfg.fill_(cfg_strength)
try:
for _ in range(3):
if uses_cfg:
assert static_cfg is not None
self._run_cfg(*static_inputs, cfg_strength=static_cfg)
else:
self._run(*static_inputs)
# CFG warm-up populates AuKTransformer's Python-side projected-text
# cache. Clear it before capture so the graph includes
# project_text(static_text), rather than closing over the first
# request's projection and ignoring later static_text updates.
self.dit.clear_cache()
if static_cfg is not None:
static_cfg.fill_(cfg_strength)
if self._pool_handle is None:
self._pool_handle = torch.cuda.graph_pool_handle()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, pool=self._pool_handle):
# Capture the actual CFG path. The scalar value remains a
# mutable buffer, so cfg=2 and cfg=3 share this graph.
if uses_cfg:
assert static_cfg is not None
static_out = self._run_cfg(*static_inputs, cfg_strength=static_cfg)
else:
static_out = self._run(*static_inputs)
finally:
self.dit.clear_cache()
logger.info(
"Captured AuK DiT single-step CUDA graph: target_frames=%d text_tokens=%d ref_frames=%d cfg=%s",
inputs[0].shape[1],
inputs[2].shape[1],
inputs[4].shape[1],
cfg_strength,
)
return _GraphEntry(
graph=graph,
static_x=static_inputs[0],
static_x_mask=static_inputs[1],
static_text=static_inputs[2],
static_c_mask=static_inputs[3],
static_ref=static_inputs[4],
static_ref_mask=static_inputs[5],
static_timestep=static_inputs[6],
static_cfg=static_cfg,
static_out=static_out,
)
__all__ = ["AuKCUDAGraphWrapper"]
@@ -27,6 +27,7 @@ from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
from vllm_omni.diffusion.models.auk.vae_cudagraph import AuKVAEDecodeGraph
from vllm_omni.diffusion.models.interface import (
SupportAudioInput,
@@ -139,6 +140,9 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
super().__init__()
del prefix # Weights are not namespaced: one checkpoint, one pipeline.
self.od_config = od_config
max_dit_graphs = od_config.model_config.get("max_dit_graphs", 32)
if isinstance(max_dit_graphs, bool) or not isinstance(max_dit_graphs, int) or max_dit_graphs < 1:
raise ValueError("AuK max_dit_graphs must be a positive integer")
self.device = get_local_device()
self.dtype = getattr(od_config, "dtype", None) or torch.bfloat16
@@ -183,6 +187,9 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
self.dit.load_state_dict(_read_dit_weights(model_dir, self.dtype), strict=True)
self.dit = self.dit.to(device=self.device).eval()
self.dit.requires_grad_(False)
self.cudagraph_wrapper = AuKCUDAGraphWrapper(
self.dit, enabled=not od_config.enforce_eager, max_graphs=max_dit_graphs
)
# The compiled decode buckets are warmed by setup_compile(), which the
# model runner calls at startup unless the stage is enforce_eager. The
# bucket list and the plain-graph cache size come from the stage's
@@ -462,6 +469,7 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
device=self.device,
dtype=torch.float32,
generator=generator,
sampler=self.cudagraph_wrapper,
)
latents = latents.float()