[Feat][Perf][AuK]Support Online Serving and DiT CUDAGraph (#7469)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
22 files changed
+1473
-38
No files matched your search
@@ -28,13 +28,13 @@ Deliberate differences from the reference, all inference-only:
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
__all__ = ["AuKTransformer", "dit_state_dict", "sample_latents"]
|
||||
__all__ = ["AuKTransformer", "build_time_grid", "dit_state_dict", "sample_latents"]
|
||||
|
||||
|
||||
def _sdpa(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor:
|
||||
@@ -86,14 +86,17 @@ class Rotary(nn.Module):
|
||||
exponents = torch.arange(0, self.dim, 2, device=device, dtype=torch.float32) / self.dim
|
||||
return 1.0 / (self.base**exponents)
|
||||
|
||||
def forward(self, seq_len: int) -> torch.Tensor:
|
||||
"""Return frequencies ``[1, 1, seq_len, dim]`` for positions ``0..seq_len-1``."""
|
||||
def forward(self, seq_len: int, mask: torch.Tensor | None = None) -> torch.Tensor:
|
||||
"""Return rotary frequencies, compressing positions across padding."""
|
||||
inv_freq = self.inv_freq
|
||||
if inv_freq.dtype != torch.float32:
|
||||
inv_freq = self._frequencies(inv_freq.device)
|
||||
pos = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32)
|
||||
freqs = torch.outer(pos, inv_freq)
|
||||
return torch.stack((freqs, freqs), dim=-1).flatten(-2)[None, None]
|
||||
if mask is None:
|
||||
pos = torch.arange(seq_len, device=inv_freq.device, dtype=torch.float32)[None]
|
||||
else:
|
||||
pos = (mask.to(torch.int32).cumsum(dim=1) - 1).clamp_min(0).to(torch.float32)
|
||||
freqs = pos.unsqueeze(-1) * inv_freq
|
||||
return torch.stack((freqs, freqs), dim=-1).flatten(-2).unsqueeze(1)
|
||||
|
||||
|
||||
class TimeEmbedding(nn.Module):
|
||||
@@ -541,15 +544,15 @@ class AuKTransformer(nn.Module):
|
||||
|
||||
seq_len = x.shape[1]
|
||||
text_len = c.shape[1]
|
||||
rope_audio = self.rotary_embed(seq_len)
|
||||
rope_text = self.rotary_embed(text_len)
|
||||
rope_audio = self.rotary_embed(seq_len, audio_mask)
|
||||
rope_text = self.rotary_embed(text_len, c_mask)
|
||||
|
||||
for block in self.transformer_blocks:
|
||||
c, x = block(x, c, t, mask=audio_mask, rope=rope_audio, c_rope=rope_text, c_mask=c_mask)
|
||||
|
||||
x = torch.cat([c, x], dim=1)
|
||||
rope = self.rotary_embed(text_len + seq_len)
|
||||
single_mask = None if audio_mask is None else torch.cat([c_mask, audio_mask], dim=1)
|
||||
rope = self.rotary_embed(text_len + seq_len, single_mask)
|
||||
|
||||
for block in self.single_transformer_blocks:
|
||||
x = block(x, t, mask=single_mask, rope=rope)
|
||||
@@ -571,6 +574,31 @@ def dit_state_dict(
|
||||
return {name[len(prefix) :]: tensor for name, tensor in items if name.startswith(prefix)}
|
||||
|
||||
|
||||
def build_time_grid(
|
||||
*,
|
||||
nfe: int,
|
||||
sway_sampling_coef: float | None,
|
||||
t_grid: list[float] | None,
|
||||
device: torch.device | str,
|
||||
) -> torch.Tensor:
|
||||
"""Build and validate the Euler schedule before a graph replay."""
|
||||
if t_grid is not None:
|
||||
timesteps = torch.tensor(t_grid, device="cpu", dtype=torch.float32)
|
||||
else:
|
||||
timesteps = torch.linspace(0, 1, nfe + 1, device="cpu", dtype=torch.float32)
|
||||
if sway_sampling_coef is not None:
|
||||
timesteps = timesteps + sway_sampling_coef * (torch.cos(math.pi / 2 * timesteps) - 1 + timesteps)
|
||||
if (
|
||||
timesteps.ndim != 1
|
||||
or timesteps.numel() < 2
|
||||
or not bool(torch.isfinite(timesteps).all() & torch.all(timesteps[1:] > timesteps[:-1]))
|
||||
):
|
||||
raise ValueError(
|
||||
f"AuK sampling needs a strictly increasing time grid with at least two points; got {timesteps.tolist()}"
|
||||
)
|
||||
return timesteps.to(device)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def sample_latents(
|
||||
dit: AuKTransformer,
|
||||
@@ -589,6 +617,7 @@ def sample_latents(
|
||||
latent_dim: int | None = None,
|
||||
device: torch.device | str | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
sampler: Callable[..., torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Integrate the flow from noise to audio latents with explicit Euler steps.
|
||||
|
||||
@@ -640,25 +669,61 @@ def sample_latents(
|
||||
.unsqueeze(0)
|
||||
)
|
||||
|
||||
if t_grid is not None:
|
||||
t = torch.tensor(t_grid, device=device, dtype=torch.float32)
|
||||
else:
|
||||
t = torch.linspace(0, 1, nfe + 1, device=device, dtype=torch.float32)
|
||||
if sway_sampling_coef is not None:
|
||||
t = t + sway_sampling_coef * (torch.cos(math.pi / 2 * t) - 1 + t)
|
||||
t = build_time_grid(
|
||||
nfe=nfe,
|
||||
sway_sampling_coef=sway_sampling_coef,
|
||||
t_grid=t_grid,
|
||||
device=device,
|
||||
)
|
||||
if sampler is not None:
|
||||
try:
|
||||
for i in range(t.shape[0] - 1):
|
||||
velocity = sampler(
|
||||
x=x,
|
||||
text=text,
|
||||
c_mask=c_mask,
|
||||
ref=ref,
|
||||
ref_mask=ref_mask,
|
||||
timestep=t[i],
|
||||
cfg_strength=cfg_strength,
|
||||
)
|
||||
x = x + (t[i + 1] - t[i]) * velocity
|
||||
finally:
|
||||
dit.clear_cache()
|
||||
return x
|
||||
return _sample_latents(
|
||||
dit,
|
||||
initial_latents=x,
|
||||
text=text,
|
||||
c_mask=c_mask,
|
||||
ref=ref,
|
||||
ref_mask=ref_mask,
|
||||
timesteps=t,
|
||||
cfg_strength=cfg_strength,
|
||||
)
|
||||
|
||||
|
||||
def _sample_latents(
|
||||
dit: AuKTransformer,
|
||||
*,
|
||||
initial_latents: torch.Tensor,
|
||||
text: torch.Tensor,
|
||||
c_mask: torch.Tensor | None,
|
||||
ref: torch.Tensor,
|
||||
ref_mask: torch.Tensor | None,
|
||||
timesteps: torch.Tensor,
|
||||
cfg_strength: float,
|
||||
) -> torch.Tensor:
|
||||
"""Euler integration shared by eager sampling and CUDA graph capture."""
|
||||
x = initial_latents
|
||||
guided = cfg_strength >= 1e-5
|
||||
if t.numel() < 2 or not bool(torch.all(t[1:] > t[:-1])):
|
||||
raise ValueError(
|
||||
f"AuK sampling needs a strictly increasing time grid with at least two points; got {t.tolist()}"
|
||||
)
|
||||
try:
|
||||
for i in range(t.shape[0] - 1):
|
||||
for i in range(timesteps.shape[0] - 1):
|
||||
if guided:
|
||||
pred = dit(
|
||||
x,
|
||||
text,
|
||||
t[i],
|
||||
timesteps[i],
|
||||
c_mask=c_mask,
|
||||
ref=ref,
|
||||
ref_mask=ref_mask,
|
||||
@@ -668,8 +733,8 @@ def sample_latents(
|
||||
v_cond, v_uncond = pred.chunk(2, dim=0)
|
||||
v = v_cond + (v_cond - v_uncond) * cfg_strength
|
||||
else:
|
||||
v = dit(x, text, t[i], c_mask=c_mask, ref=ref, ref_mask=ref_mask)
|
||||
x = x + (t[i + 1] - t[i]) * v
|
||||
v = dit(x, text, timesteps[i], c_mask=c_mask, ref=ref, ref_mask=ref_mask)
|
||||
x = x + (timesteps[i + 1] - timesteps[i]) * v
|
||||
finally:
|
||||
# The cached text projections belong to this request only; a failed
|
||||
# step must not leak them into the next one.
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Per-shape CUDA graph for one AuK DiT denoise step."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _GraphEntry:
|
||||
graph: torch.cuda.CUDAGraph
|
||||
static_x: torch.Tensor
|
||||
static_x_mask: torch.Tensor
|
||||
static_text: torch.Tensor
|
||||
static_c_mask: torch.Tensor
|
||||
static_ref: torch.Tensor
|
||||
static_ref_mask: torch.Tensor
|
||||
static_timestep: torch.Tensor
|
||||
static_cfg: torch.Tensor | None
|
||||
static_out: torch.Tensor
|
||||
|
||||
|
||||
class AuKCUDAGraphWrapper:
|
||||
"""Replay one DiT denoise step and leave Euler scheduling to the caller.
|
||||
|
||||
The graph is keyed by the target, text and reference sequence lengths plus
|
||||
whether the CFG branch is enabled. Timestep and the CFG strength are
|
||||
mutable scalar buffers, so all Euler steps and CFG values within one path
|
||||
reuse one graph.
|
||||
"""
|
||||
|
||||
_TARGET_ALIGNMENT = 64
|
||||
_TEXT_ALIGNMENT = 64
|
||||
_REF_ALIGNMENT = 50
|
||||
|
||||
def __init__(self, dit: AuKTransformer, *, enabled: bool = True, max_graphs: int = 32) -> None:
|
||||
self.dit = dit
|
||||
self.enabled = bool(enabled)
|
||||
self.max_graphs = max(1, int(max_graphs))
|
||||
self._cache: OrderedDict[tuple, _GraphEntry] = OrderedDict()
|
||||
self._pool_handle: int | None = None
|
||||
logger.info("Initialized AuK DiT lazy CUDA graph cache: enabled=%s slots=%d", self.enabled, self.max_graphs)
|
||||
|
||||
@classmethod
|
||||
def _bucket_inputs(
|
||||
cls,
|
||||
x: torch.Tensor,
|
||||
text: torch.Tensor,
|
||||
c_mask: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
ref_mask: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
target_bucket = round_up(x.shape[1], cls._TARGET_ALIGNMENT)
|
||||
text_bucket = round_up(text.shape[1], cls._TEXT_ALIGNMENT)
|
||||
ref_bucket = round_up(ref.shape[1], cls._REF_ALIGNMENT)
|
||||
x_mask = torch.ones(x.shape[:2], dtype=torch.bool, device=x.device)
|
||||
return (
|
||||
F.pad(x, (0, 0, 0, target_bucket - x.shape[1])),
|
||||
F.pad(x_mask, (0, target_bucket - x_mask.shape[1]), value=False),
|
||||
F.pad(text, (0, 0, 0, text_bucket - text.shape[1])),
|
||||
F.pad(c_mask, (0, text_bucket - c_mask.shape[1]), value=False),
|
||||
F.pad(ref, (0, 0, 0, ref_bucket - ref.shape[1])),
|
||||
F.pad(ref_mask, (0, ref_bucket - ref_mask.shape[1]), value=False),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _key(
|
||||
x: torch.Tensor,
|
||||
text: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
uses_cfg: bool,
|
||||
) -> tuple[int, int, int, bool]:
|
||||
return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg)
|
||||
|
||||
def _retire_graph_generation_if_full(self) -> None:
|
||||
"""Retire all graphs together so none outlive shared workspaces."""
|
||||
if len(self._cache) >= self.max_graphs:
|
||||
self._cache.clear()
|
||||
|
||||
@torch.no_grad()
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
x: torch.Tensor,
|
||||
text: torch.Tensor,
|
||||
c_mask: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
ref_mask: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
cfg_strength: float,
|
||||
) -> torch.Tensor:
|
||||
uses_cfg = cfg_strength >= 1e-5
|
||||
if not self.enabled or x.device.type != "cuda" or torch.cuda.is_current_stream_capturing():
|
||||
if uses_cfg:
|
||||
return self._run_cfg(x, None, text, c_mask, ref, ref_mask, timestep, cfg_strength=cfg_strength)
|
||||
return self._run(x, None, text, c_mask, ref, ref_mask, timestep)
|
||||
|
||||
target_frames = x.shape[1]
|
||||
x, x_mask, text, c_mask, ref, ref_mask = self._bucket_inputs(x, text, c_mask, ref, ref_mask)
|
||||
inputs = (x, x_mask, text, c_mask, ref, ref_mask, timestep)
|
||||
key = self._key(x, text, ref, uses_cfg)
|
||||
entry = self._cache.get(key)
|
||||
if entry is None:
|
||||
# Same Retirement as #6587, details discussed in #7469
|
||||
self._retire_graph_generation_if_full()
|
||||
try:
|
||||
entry = self._capture(*inputs, cfg_strength=cfg_strength, uses_cfg=uses_cfg)
|
||||
except Exception:
|
||||
self._cache.clear()
|
||||
self.enabled = False
|
||||
logger.exception(
|
||||
"Disabling AuK DiT CUDA graphs after capture failure for key=%s; "
|
||||
"subsequent requests will use eager execution.",
|
||||
key,
|
||||
)
|
||||
raise
|
||||
self._cache[key] = entry
|
||||
else:
|
||||
self._cache.move_to_end(key)
|
||||
|
||||
entry.static_x.copy_(x)
|
||||
entry.static_x_mask.copy_(x_mask)
|
||||
entry.static_text.copy_(text)
|
||||
entry.static_c_mask.copy_(c_mask)
|
||||
entry.static_ref.copy_(ref)
|
||||
entry.static_ref_mask.copy_(ref_mask)
|
||||
entry.static_timestep.copy_(timestep)
|
||||
if entry.static_cfg is not None:
|
||||
entry.static_cfg.fill_(cfg_strength)
|
||||
entry.graph.replay()
|
||||
return entry.static_out[:, :target_frames].clone()
|
||||
|
||||
def _run(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
x_mask: torch.Tensor | None,
|
||||
text: torch.Tensor,
|
||||
c_mask: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
ref_mask: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.dit(x, text, timestep, mask=x_mask, c_mask=c_mask, ref=ref, ref_mask=ref_mask)
|
||||
|
||||
def _run_cfg(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
x_mask: torch.Tensor | None,
|
||||
text: torch.Tensor,
|
||||
c_mask: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
ref_mask: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
*,
|
||||
cfg_strength: torch.Tensor | float,
|
||||
) -> torch.Tensor:
|
||||
pred = self.dit(
|
||||
x,
|
||||
text,
|
||||
timestep,
|
||||
mask=x_mask,
|
||||
c_mask=c_mask,
|
||||
ref=ref,
|
||||
ref_mask=ref_mask,
|
||||
cfg_infer=True,
|
||||
cache=True,
|
||||
)
|
||||
conditional, unconditional = pred.chunk(2, dim=0)
|
||||
return conditional + (conditional - unconditional) * cfg_strength
|
||||
|
||||
def _capture(
|
||||
self,
|
||||
*inputs: torch.Tensor,
|
||||
cfg_strength: float,
|
||||
uses_cfg: bool,
|
||||
) -> _GraphEntry:
|
||||
static_inputs = tuple(value.clone() for value in inputs)
|
||||
static_cfg = None
|
||||
if uses_cfg:
|
||||
static_cfg = torch.empty((), device=static_inputs[0].device, dtype=torch.float32)
|
||||
static_cfg.fill_(cfg_strength)
|
||||
try:
|
||||
for _ in range(3):
|
||||
if uses_cfg:
|
||||
assert static_cfg is not None
|
||||
self._run_cfg(*static_inputs, cfg_strength=static_cfg)
|
||||
else:
|
||||
self._run(*static_inputs)
|
||||
# CFG warm-up populates AuKTransformer's Python-side projected-text
|
||||
# cache. Clear it before capture so the graph includes
|
||||
# project_text(static_text), rather than closing over the first
|
||||
# request's projection and ignoring later static_text updates.
|
||||
self.dit.clear_cache()
|
||||
if static_cfg is not None:
|
||||
static_cfg.fill_(cfg_strength)
|
||||
if self._pool_handle is None:
|
||||
self._pool_handle = torch.cuda.graph_pool_handle()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, pool=self._pool_handle):
|
||||
# Capture the actual CFG path. The scalar value remains a
|
||||
# mutable buffer, so cfg=2 and cfg=3 share this graph.
|
||||
if uses_cfg:
|
||||
assert static_cfg is not None
|
||||
static_out = self._run_cfg(*static_inputs, cfg_strength=static_cfg)
|
||||
else:
|
||||
static_out = self._run(*static_inputs)
|
||||
finally:
|
||||
self.dit.clear_cache()
|
||||
|
||||
logger.info(
|
||||
"Captured AuK DiT single-step CUDA graph: target_frames=%d text_tokens=%d ref_frames=%d cfg=%s",
|
||||
inputs[0].shape[1],
|
||||
inputs[2].shape[1],
|
||||
inputs[4].shape[1],
|
||||
cfg_strength,
|
||||
)
|
||||
return _GraphEntry(
|
||||
graph=graph,
|
||||
static_x=static_inputs[0],
|
||||
static_x_mask=static_inputs[1],
|
||||
static_text=static_inputs[2],
|
||||
static_c_mask=static_inputs[3],
|
||||
static_ref=static_inputs[4],
|
||||
static_ref_mask=static_inputs[5],
|
||||
static_timestep=static_inputs[6],
|
||||
static_cfg=static_cfg,
|
||||
static_out=static_out,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["AuKCUDAGraphWrapper"]
|
||||
@@ -27,6 +27,7 @@ from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
|
||||
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
|
||||
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
|
||||
from vllm_omni.diffusion.models.auk.vae_cudagraph import AuKVAEDecodeGraph
|
||||
from vllm_omni.diffusion.models.interface import (
|
||||
SupportAudioInput,
|
||||
@@ -139,6 +140,9 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
|
||||
super().__init__()
|
||||
del prefix # Weights are not namespaced: one checkpoint, one pipeline.
|
||||
self.od_config = od_config
|
||||
max_dit_graphs = od_config.model_config.get("max_dit_graphs", 32)
|
||||
if isinstance(max_dit_graphs, bool) or not isinstance(max_dit_graphs, int) or max_dit_graphs < 1:
|
||||
raise ValueError("AuK max_dit_graphs must be a positive integer")
|
||||
self.device = get_local_device()
|
||||
self.dtype = getattr(od_config, "dtype", None) or torch.bfloat16
|
||||
|
||||
@@ -183,6 +187,9 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
|
||||
self.dit.load_state_dict(_read_dit_weights(model_dir, self.dtype), strict=True)
|
||||
self.dit = self.dit.to(device=self.device).eval()
|
||||
self.dit.requires_grad_(False)
|
||||
self.cudagraph_wrapper = AuKCUDAGraphWrapper(
|
||||
self.dit, enabled=not od_config.enforce_eager, max_graphs=max_dit_graphs
|
||||
)
|
||||
# The compiled decode buckets are warmed by setup_compile(), which the
|
||||
# model runner calls at startup unless the stage is enforce_eager. The
|
||||
# bucket list and the plain-graph cache size come from the stage's
|
||||
@@ -462,6 +469,7 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
|
||||
device=self.device,
|
||||
dtype=torch.float32,
|
||||
generator=generator,
|
||||
sampler=self.cudagraph_wrapper,
|
||||
)
|
||||
|
||||
latents = latents.float()
|
||||
|
||||
Reference in new issue
Block a user