[Perf][AuK] Precompute adaLN modulations per request and run the conv position embedding as one GEMM (#8328)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-10-02 08:12:29 -07:00
1 parent 0e6791e9c4
commit 13c66da752
6 files changed
+329 -45

No files matched your search

@@ -61,7 +61,10 @@ def test_single_request_graph_wrapper_cpu_falls_back_to_eager(cfg_strength: floa
# The conditioning is prepared once per request and reused by every step.
assert prepare_spy.call_count == 1
assert prepare_spy.call_args.args[-1] is (cfg_strength >= 1e-5)
# _prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg, timesteps)
assert prepare_spy.call_args.args[6] is (cfg_strength >= 1e-5)
# The time grid reaches prepare, so the adaLN modulations are computed once.
assert prepare_spy.call_args.args[7].numel() == 2
assert step_spy.call_count == 2
capture_spy.assert_not_called()
assert not wrapper._cache
@@ -84,7 +87,8 @@ def test_graph_inputs_use_bounded_length_buckets() -> None:
assert padded[4].shape == (1, 100, 4)
assert padded[5].shape == (1, 100)
assert [mask.sum().item() for mask in (padded[1], padded[3], padded[5])] == [65, 65, 51]
assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False)
assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False, 0)
assert wrapper._key(padded[0], padded[2], padded[4], False, 32) == (96, 96, 100, False, 32)
assert wrapper.max_graphs == 32
@@ -153,18 +157,21 @@ def test_single_request_graph_replay_matches_eager_and_updates_inputs(cfg_streng
inputs["ref"],
inputs["ref_mask"],
)
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5)
# sample_latents passes the grid, so the key carries its step count (two steps here).
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5, 2)
assert key in wrapper._cache
# The static context was refreshed for this request's conditioning.
entry = wrapper._cache[key]
fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5)
grid = torch.tensor([0.0, 0.4], device="cuda")
fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5, grid)
for static, want in zip(entry.static_ctx.tensors(), fresh.tensors(), strict=True):
if want is None:
assert static is None
else:
torch.testing.assert_close(static, want)
torch.testing.assert_close(entry.static_timestep, torch.tensor(0.4, device="cuda"))
assert int(entry.static_step) == 1
assert len(wrapper._cache) == 1
@@ -0,0 +1,139 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The AuK DiT fast paths keep the reference numerics.
Two paths are checked: the grouped conv position embedding as one batched
GEMM, and the adaLN modulations precomputed for a whole time grid, eager and
under the per-step CUDA graph.
"""
import pytest
import torch
from vllm_omni.diffusion.models.auk import auk_transformer as T
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, build_time_grid
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_dit(device: str = "cpu") -> AuKTransformer:
torch.manual_seed(5)
return (
AuKTransformer(
dim=32,
heads=2,
dim_head=16,
ff_mult=2,
latent_dim=4,
text_hidden_dim=8,
num_layers=2,
num_single_layers=2,
)
.eval()
.to(device)
)
def _reference_conv_pos_embed(module: T.ConvPosEmbedding, x: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor:
"""The channels-first conv stack as the reference implementation writes it."""
keep = None if mask is None else mask.unsqueeze(1)
x = x.transpose(1, 2)
if keep is not None:
x = x.masked_fill(~keep, 0.0)
for layer in module.conv1d:
x = layer(x)
if keep is not None and isinstance(layer, torch.nn.Conv1d):
x = x.masked_fill(~keep, 0.0)
return x.transpose(1, 2)
@torch.inference_mode()
def test_grouped_conv_as_batched_gemm_matches_conv1d() -> None:
torch.manual_seed(0)
conv = torch.nn.Conv1d(64, 64, 31, groups=16, padding=15)
x = torch.randn(2, 37, 64)
expected = conv(x.transpose(1, 2)).transpose(1, 2)
torch.testing.assert_close(T._grouped_conv1d_btc(x, conv), expected, atol=1e-5, rtol=1e-5)
@torch.inference_mode()
@pytest.mark.parametrize("masked", [False, True])
def test_conv_pos_embedding_matches_the_channels_first_stack(masked: bool) -> None:
torch.manual_seed(1)
module = T.ConvPosEmbedding(64).eval()
x = torch.randn(2, 37, 64)
mask = None
if masked:
mask = torch.ones(2, 37, dtype=torch.bool)
mask[1, 30:] = False
torch.testing.assert_close(module(x, mask), _reference_conv_pos_embed(module, x, mask), atol=1e-5, rtol=1e-5)
@torch.inference_mode()
@pytest.mark.parametrize("cfg_infer", [False, True])
def test_modulation_table_matches_per_step_modulation(cfg_infer: bool) -> None:
dit = _make_dit()
text = torch.randn(1, 5, 8)
ref = torch.randn(1, 3, 4)
x = torch.randn(1, 6, 4)
grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cpu")
plain = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer)
tabled = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer, timesteps=grid[:-1])
assert tabled.modulation is not None and tabled.modulation.shape[0] == 4
for i in range(4):
expected = dit.step(x, grid[i], plain)
torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=i), expected, atol=1e-5, rtol=1e-5)
index = torch.tensor(i)
torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=index), expected, atol=1e-5, rtol=1e-5)
def _padded_inputs(device: str) -> dict[str, torch.Tensor]:
generator = torch.Generator(device="cpu").manual_seed(3)
text = torch.randn(1, 7, 8, generator=generator)
c_mask = torch.ones(1, 7, dtype=torch.bool)
c_mask[:, 5:] = False
text[:, 5:] = 0.0
ref = torch.randn(1, 6, 4, generator=generator)
ref_mask = torch.ones(1, 6, dtype=torch.bool)
ref_mask[:, 4:] = False
mask = torch.ones(1, 9, dtype=torch.bool)
mask[:, 7:] = False
inputs = {"text": text, "c_mask": c_mask, "ref": ref, "ref_mask": ref_mask, "mask": mask}
return {name: value.to(device) for name, value in inputs.items()}
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA")
@torch.inference_mode()
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
def test_graph_with_time_grid_matches_eager_with_time_grid(cfg_strength: float) -> None:
dit = _make_dit("cuda")
inputs = _padded_inputs("cuda")
grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cuda")
x = torch.randn(1, 9, 4, device="cuda")
wrapper = AuKCUDAGraphWrapper(dit)
common = dict(
text=inputs["text"],
c_mask=inputs["c_mask"],
ref=inputs["ref"],
ref_mask=inputs["ref_mask"],
cfg_strength=cfg_strength,
timesteps=grid[:-1],
)
for i in range(4):
graph = wrapper(x=x, timestep=grid[i], new_request=i == 0, step_index=i, **common)
ctx = dit.prepare(
inputs["text"],
target_len=9,
c_mask=inputs["c_mask"],
ref=inputs["ref"],
ref_mask=inputs["ref_mask"],
cfg_infer=cfg_strength >= 1e-5,
timesteps=grid[:-1],
)
eager = wrapper._step(x, grid[i], ctx, cfg_strength, i)
torch.testing.assert_close(graph, eager, atol=1e-4, rtol=1e-4)
# One graph serves every step of the grid.
assert len(wrapper._cache) == 1
assert next(iter(wrapper._cache))[-1] == 4
@@ -127,7 +127,9 @@ def test_hoisted_euler_loop_matches_per_step_forward(cfg_strength: float) -> Non
v = dit(x, time=timesteps[i], **inputs)
x = x + (timesteps[i + 1] - timesteps[i]) * v
torch.testing.assert_close(got, x, rtol=0, atol=0)
# sample_latents precomputes the adaLN modulations of the whole grid in one
# GEMM per layer, whose rows round differently from one-row GEMMs in fp32.
torch.testing.assert_close(got, x, rtol=0, atol=1e-6)
assert dit.text_cond is None and dit.text_uncond is None
+126 -26
View File
@@ -153,6 +153,33 @@ class TimeEmbedding(nn.Module):
return self.time_mlp(hidden.to(self.time_mlp[0].weight.dtype))
def _grouped_conv1d_btc(x: torch.Tensor, conv: nn.Conv1d) -> torch.Tensor:
"""``conv`` applied to channels-last ``x`` ``[B, T, C]``, returning ``[B, T, C]``.
A same-padded grouped convolution written as one im2col copy and one
batched GEMM over the groups. cuDNN runs a grouped conv of this shape as
one kernel per group plus two layout transposes, which at AuK's sizes
(1536 channels in 16 groups, kernel 31, a few hundred frames) costs about
ten times the arithmetic.
"""
batch, frames, channels = x.shape
groups = conv.groups
kernel = conv.kernel_size[0]
per_group = channels // groups
pad = conv.padding[0]
padded = F.pad(x, (0, 0, pad, pad))
# unfold gives a [B, T, C, K] view; one copy lays it out as [B, T, G, C/G * K].
cols = padded.unfold(1, kernel, 1).reshape(batch * frames, groups, per_group * kernel)
# conv.weight [C_out, C/G, K] -> [G, C/G * K, C_out/G]; the reduction order matches the columns.
weight = conv.weight.reshape(groups, channels // groups, per_group * kernel).transpose(1, 2).to(cols.dtype)
# [G, B*T, C/G*K] @ [G, C/G*K, C_out/G] -> [G, B*T, C_out/G]
out = torch.bmm(cols.transpose(0, 1), weight)
out = out.transpose(0, 1).reshape(batch, frames, channels)
if conv.bias is not None:
out = out + conv.bias.to(out.dtype)
return out
class ConvPosEmbedding(nn.Module):
"""Depthwise-grouped conv stack that adds local position information."""
@@ -169,17 +196,19 @@ class ConvPosEmbedding(nn.Module):
)
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
keep = None if mask is None else mask.unsqueeze(1)
x = x.transpose(1, 2)
keep = None if mask is None else mask.unsqueeze(-1)
if keep is not None:
x = x.masked_fill(~keep, 0.0)
for layer in self.conv1d:
x = layer(x)
# Re-zero padding after each convolution so it cannot bleed into
# valid frames through the kernel window.
if keep is not None and isinstance(layer, nn.Conv1d):
x = x.masked_fill(~keep, 0.0)
return x.transpose(1, 2)
if isinstance(layer, nn.Conv1d):
x = _grouped_conv1d_btc(x, layer)
# Re-zero padding after each convolution so it cannot bleed into
# valid frames through the kernel window.
if keep is not None:
x = x.masked_fill(~keep, 0.0)
else:
x = layer(x)
return x
class AudioEmbedding(nn.Module):
@@ -205,9 +234,12 @@ class AdaLayerNorm(nn.Module):
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
def forward(
self, x: torch.Tensor, emb: torch.Tensor
self, x: torch.Tensor, emb: torch.Tensor | None, mod: torch.Tensor | None = None
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.linear(self.silu(emb)).chunk(6, dim=1)
"""``mod`` is the precomputed ``linear(silu(emb))``; ``emb`` is then unused."""
if mod is None:
mod = self.linear(self.silu(emb))
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=1)
x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
return x, gate_msa, shift_mlp, scale_mlp, gate_mlp
@@ -221,8 +253,11 @@ class AdaLayerNormFinal(nn.Module):
self.linear = nn.Linear(dim, dim * 2)
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor:
scale, shift = self.linear(self.silu(emb)).chunk(2, dim=1)
def forward(self, x: torch.Tensor, emb: torch.Tensor | None, mod: torch.Tensor | None = None) -> torch.Tensor:
"""``mod`` is the precomputed ``linear(silu(emb))``; ``emb`` is then unused."""
if mod is None:
mod = self.linear(self.silu(emb))
scale, shift = mod.chunk(2, dim=1)
return self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
@@ -383,9 +418,11 @@ class DoubleBlock(nn.Module):
c_rope: RopeCache,
c_mask: torch.Tensor | None,
bias: torch.Tensor | None = None,
mod_c: torch.Tensor | None = None,
mod_x: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t)
norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t)
norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t, mod_c)
norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t, mod_x)
x_attn, c_attn = self.attn(x=norm_x, c=norm_c, mask=mask, rope=rope, c_rope=c_rope, c_mask=c_mask, bias=bias)
@@ -416,8 +453,9 @@ class SingleBlock(nn.Module):
mask: torch.Tensor | None,
rope: RopeCache,
bias: torch.Tensor | None = None,
mod: torch.Tensor | None = None,
) -> torch.Tensor:
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t)
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t, mod)
x = x + gate_msa[:, None] * self.attn(x=norm, mask=mask, rope=rope, bias=bias)
norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
return x + gate_mlp[:, None] * self.ff(norm)
@@ -491,6 +529,28 @@ class AuKTransformer(nn.Module):
self.norm_out = AdaLayerNormFinal(dim)
self.proj_out = nn.Linear(dim, latent_dim)
# Widths of the adaLN modulations in the order modulation_table() lays them out.
self._modulation_widths = [module.linear.out_features for module in self._adaln_modules()]
def _adaln_modules(self) -> list[nn.Module]:
"""Every timestep-conditioned norm, in the order :meth:`step` consumes them."""
modules: list[nn.Module] = []
for block in self.transformer_blocks:
modules += [block.attn_norm_c, block.attn_norm_x]
modules += [block.attn_norm for block in self.single_transformer_blocks]
modules.append(self.norm_out)
return modules
def modulation_table(self, timesteps: torch.Tensor) -> torch.Tensor:
"""adaLN modulations for every timestep, ``[len(timesteps), sum of widths]``.
The modulations depend only on the timestep, and a request's time grid
is known before its first step. Computing them for the whole grid at
once reads each modulation weight once per request instead of once per
step, which at batch 1 is what those matrix-vector products cost.
"""
emb = F.silu(self.time_embed(timesteps.reshape(-1).float()))
return torch.cat([module.linear(emb) for module in self._adaln_modules()], dim=1)
def project_text(self, text: torch.Tensor) -> torch.Tensor:
"""Project LLM hidden states ``[B, nt, text_hidden_dim]`` to model width."""
@@ -520,6 +580,7 @@ class AuKTransformer(nn.Module):
drop_text: bool = False,
cfg_infer: bool = False,
cache: bool = False,
timesteps: torch.Tensor | None = None,
) -> "AuKStepContext":
"""Everything a denoise step needs that does not depend on ``x`` or the time.
@@ -528,6 +589,9 @@ class AuKTransformer(nn.Module):
prompt embedding, the padding masks and biases and the rotary tables
are not recomputed on every step. Arguments are those of
:meth:`forward`; ``target_len`` is the target frame count ``n``.
``timesteps`` are the times the steps will run at (the grid without
its end point); when given, their adaLN modulations are precomputed
and :meth:`step` selects them by ``step_index``.
"""
batch = text.shape[0]
if c_mask is None:
@@ -593,25 +657,44 @@ class AuKTransformer(nn.Module):
rope_text=_rope_cos_sin(self.rotary_embed(text_len, c_mask)),
rope_single=_rope_cos_sin(self.rotary_embed(text_len + audio_len, single_mask)),
branches=branches,
modulation=None if timesteps is None else self.modulation_table(timesteps),
)
def step(self, x: torch.Tensor, time: torch.Tensor, ctx: "AuKStepContext") -> torch.Tensor:
def step(
self,
x: torch.Tensor,
time: torch.Tensor,
ctx: "AuKStepContext",
step_index: torch.Tensor | int | None = None,
) -> torch.Tensor:
"""Predict the velocity of ``x`` at ``time`` under a :meth:`prepare` context.
The target embedding does not depend on the CFG branch, so it is
computed once and shared by both branches.
computed once and shared by both branches. When the context carries a
modulation table, ``step_index`` selects this step's row of it and
``time`` is unused; the row broadcasts over the CFG branches.
"""
if time.ndim == 0:
time = time.repeat(x.shape[0])
t = self.time_embed(time)
target = self.audio_embed(x, mask=ctx.target_mask)
if ctx.modulation is not None and step_index is not None:
if isinstance(step_index, int):
row = ctx.modulation[step_index : step_index + 1]
else:
row = ctx.modulation.index_select(0, step_index.reshape(1))
mods = list(torch.split(row, self._modulation_widths, dim=1))
t = None
else:
if time.ndim == 0:
time = time.repeat(x.shape[0])
t = self.time_embed(time)
if ctx.branches == 2:
t = torch.cat((t, t), dim=0)
mods = [None] * len(self._modulation_widths)
if ctx.branches == 2:
t = torch.cat((t, t), dim=0)
target = torch.cat((target, target), dim=0)
audio = target if ctx.prompt is None else torch.cat([ctx.prompt, target], dim=1)
c = ctx.c
for block in self.transformer_blocks:
for i, block in enumerate(self.transformer_blocks):
c, audio = block(
audio,
c,
@@ -621,14 +704,24 @@ class AuKTransformer(nn.Module):
c_rope=ctx.rope_text,
c_mask=ctx.c_mask,
bias=ctx.joint_bias,
mod_c=mods[2 * i],
mod_x=mods[2 * i + 1],
)
h = torch.cat([c, audio], dim=1)
for block in self.single_transformer_blocks:
h = block(h, t, mask=ctx.single_mask, rope=ctx.rope_single, bias=ctx.single_bias)
offset = 2 * len(self.transformer_blocks)
for i, block in enumerate(self.single_transformer_blocks):
h = block(
h,
t,
mask=ctx.single_mask,
rope=ctx.rope_single,
bias=ctx.single_bias,
mod=mods[offset + i],
)
h = h[:, h.shape[1] - x.shape[1] :]
return self.proj_out(self.norm_out(h, t))
return self.proj_out(self.norm_out(h, t, mods[-1]))
def forward(
self,
@@ -706,6 +799,8 @@ class AuKStepContext:
rope_text: RopeCache
rope_single: RopeCache
branches: int
#: adaLN modulations per step, ``[steps, widths]``, when the time grid was known.
modulation: torch.Tensor | None = None
def tensors(self) -> list[torch.Tensor | None]:
"""Every tensor field in a fixed order, rope pairs flattened."""
@@ -721,6 +816,7 @@ class AuKStepContext:
*self.rope_audio,
*self.rope_text,
*self.rope_single,
self.modulation,
]
def copy_(self, other: "AuKStepContext") -> None:
@@ -752,6 +848,7 @@ class AuKStepContext:
rope_text=(self.rope_text[0].clone(), self.rope_text[1].clone()),
rope_single=(self.rope_single[0].clone(), self.rope_single[1].clone()),
branches=self.branches,
modulation=_clone(self.modulation),
)
@@ -881,6 +978,8 @@ def sample_latents(
timestep=t[i],
cfg_strength=cfg_strength,
new_request=i == 0,
timesteps=t[:-1],
step_index=i,
)
x = x + (t[i + 1] - t[i]) * velocity
finally:
@@ -922,9 +1021,10 @@ def _sample_latents(
ref_mask=ref_mask,
cfg_infer=guided,
cache=guided,
timesteps=timesteps[:-1],
)
for i in range(timesteps.shape[0] - 1):
v = dit.step(x, timesteps[i], ctx)
v = dit.step(x, timesteps[i], ctx, step_index=i)
if guided:
v_cond, v_uncond = v.chunk(2, dim=0)
v = v_cond + (v_cond - v_uncond) * cfg_strength
@@ -24,16 +24,19 @@ class _GraphEntry:
static_ctx: AuKStepContext
static_timestep: torch.Tensor
static_cfg: torch.Tensor | None
static_step: torch.Tensor | None
static_out: torch.Tensor
class AuKCUDAGraphWrapper:
"""Replay one DiT denoise step and leave Euler scheduling to the caller.
The graph is keyed by the target, text and reference sequence lengths plus
whether the CFG branch is enabled. Timestep and the CFG strength are
The graph is keyed by the target, text and reference sequence lengths,
whether the CFG branch is enabled and, when the caller passes the time
grid, its length. Timestep (or the step index), and the CFG strength are
mutable scalar buffers, so all Euler steps and CFG values within one path
reuse one graph.
reuse one graph. With the time grid, the adaLN modulations of every step
are part of the per-request context and the graph selects its row.
Only :meth:`AuKTransformer.step` is captured. The per-request context
(text projection, reference embedding, padding biases, rotary tables) is
@@ -85,8 +88,9 @@ class AuKCUDAGraphWrapper:
text: torch.Tensor,
ref: torch.Tensor,
uses_cfg: bool,
) -> tuple[int, int, int, bool]:
return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg)
steps: int = 0,
) -> tuple[int, int, int, bool, int]:
return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg, steps)
def _retire_graph_generation_if_full(self) -> None:
"""Retire all graphs together so none outlive shared workspaces."""
@@ -103,6 +107,7 @@ class AuKCUDAGraphWrapper:
ref: torch.Tensor,
ref_mask: torch.Tensor,
uses_cfg: bool,
timesteps: torch.Tensor | None = None,
) -> AuKStepContext:
return self.dit.prepare(
text,
@@ -112,6 +117,7 @@ class AuKCUDAGraphWrapper:
ref=ref,
ref_mask=ref_mask,
cfg_infer=uses_cfg,
timesteps=timesteps,
)
def _step(
@@ -120,9 +126,10 @@ class AuKCUDAGraphWrapper:
timestep: torch.Tensor,
ctx: AuKStepContext,
cfg_strength: torch.Tensor | float,
step_index: torch.Tensor | int | None = None,
) -> torch.Tensor:
"""One guided (or unguided) velocity; the CFG combine is part of the graph."""
velocity = self.dit.step(x, timestep, ctx)
velocity = self.dit.step(x, timestep, ctx, step_index=step_index)
if ctx.branches == 1:
return velocity
conditional, unconditional = velocity.chunk(2, dim=0)
@@ -140,6 +147,8 @@ class AuKCUDAGraphWrapper:
timestep: torch.Tensor,
cfg_strength: float,
new_request: bool = True,
timesteps: torch.Tensor | None = None,
step_index: int | None = None,
) -> torch.Tensor:
"""Velocity of ``x`` at ``timestep``.
@@ -147,26 +156,33 @@ class AuKCUDAGraphWrapper:
(``text``, ``c_mask``, ``ref``, ``ref_mask``) is then prepared anew; the
later steps of the request pass ``False`` and reuse it. The default
prepares on every call, which is always correct.
``timesteps`` (the times every step of the request runs at) and
``step_index`` (this step's position in them) let the adaLN
modulations be computed once per request instead of once per step.
"""
uses_cfg = cfg_strength >= 1e-5
if timesteps is None or step_index is None:
timesteps, step_index = None, None
if not self.enabled or x.device.type != "cuda" or torch.cuda.is_current_stream_capturing():
if new_request or self._eager_ctx is None:
self._eager_ctx = self._prepare(x, None, text, c_mask, ref, ref_mask, uses_cfg)
return self._step(x, timestep, self._eager_ctx, cfg_strength)
self._eager_ctx = self._prepare(x, None, text, c_mask, ref, ref_mask, uses_cfg, timesteps)
return self._step(x, timestep, self._eager_ctx, cfg_strength, step_index)
target_frames = x.shape[1]
x, x_mask, text, c_mask, ref, ref_mask = self._bucket_inputs(x, text, c_mask, ref, ref_mask)
key = self._key(x, text, ref, uses_cfg)
steps = 0 if timesteps is None else int(timesteps.numel())
key = self._key(x, text, ref, uses_cfg, steps)
entry = self._cache.get(key)
ctx = None
if entry is None or new_request or key != self._active_key:
ctx = self._prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg)
ctx = self._prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg, timesteps)
if entry is None:
# Same Retirement as #6587, details discussed in #7469
self._retire_graph_generation_if_full()
try:
assert ctx is not None
entry = self._capture(x, ctx, timestep, cfg_strength=cfg_strength)
entry = self._capture(x, ctx, timestep, cfg_strength=cfg_strength, step_index=step_index)
except Exception:
self._cache.clear()
self._active_key = None
@@ -186,6 +202,8 @@ class AuKCUDAGraphWrapper:
entry.static_x.copy_(x)
entry.static_timestep.copy_(timestep)
if entry.static_step is not None:
entry.static_step.fill_(step_index)
if entry.static_cfg is not None:
entry.static_cfg.fill_(cfg_strength)
entry.graph.replay()
@@ -198,10 +216,14 @@ class AuKCUDAGraphWrapper:
timestep: torch.Tensor,
*,
cfg_strength: float,
step_index: int | None = None,
) -> _GraphEntry:
static_x = x.clone()
static_ctx = ctx.clone()
static_timestep = timestep.clone()
static_step = None
if static_ctx.modulation is not None and step_index is not None:
static_step = torch.full((), step_index, device=static_x.device, dtype=torch.long)
static_cfg = None
if ctx.branches == 2:
static_cfg = torch.empty((), device=static_x.device, dtype=torch.float32)
@@ -209,13 +231,13 @@ class AuKCUDAGraphWrapper:
scale = static_cfg if static_cfg is not None else cfg_strength
# Warm-up runs any lazy torch.compile outside the capture.
for _ in range(3):
self._step(static_x, static_timestep, static_ctx, scale)
self._step(static_x, static_timestep, static_ctx, scale, static_step)
if self._pool_handle is None:
self._pool_handle = torch.cuda.graph_pool_handle()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, pool=self._pool_handle):
# The CFG scalar stays a mutable buffer, so cfg=2 and cfg=3 share this graph.
static_out = self._step(static_x, static_timestep, static_ctx, scale)
static_out = self._step(static_x, static_timestep, static_ctx, scale, static_step)
logger.info(
"Captured AuK DiT single-step CUDA graph: target_frames=%d text_tokens=%d ref_frames=%d cfg=%s",
@@ -230,6 +252,7 @@ class AuKCUDAGraphWrapper:
static_ctx=static_ctx,
static_timestep=static_timestep,
static_cfg=static_cfg,
static_step=static_step,
static_out=static_out,
)
+14 -1
View File
@@ -27,7 +27,12 @@ from vllm.logger import init_logger
from vllm_omni.diffusion.compile import regionally_compile
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
from vllm_omni.diffusion.models.auk.auk_transformer import (
AuKTransformer,
build_time_grid,
dit_state_dict,
sample_latents,
)
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
from vllm_omni.diffusion.models.auk.vae_cudagraph import AuKVAEDecodeGraph
@@ -267,6 +272,12 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
text = torch.zeros(1, _WARMUP_TEXT_TOKENS, self.text_hidden_dim, device=self.device, dtype=self.dtype)
c_mask = torch.ones(text.shape[:2], dtype=torch.bool, device=self.device)
timestep = torch.zeros((), device=self.device, dtype=torch.float32)
# The graphs are keyed by the number of steps, so warm the variant's default grid.
if self.is_flash:
grid = torch.tensor(self.flash_t_grid, device=self.device, dtype=torch.float32)
else:
sway = None if self.default_sway is None else float(self.default_sway)
grid = build_time_grid(nfe=self.default_nfe, sway_sampling_coef=sway, t_grid=None, device=self.device)
start = time.perf_counter()
with torch.inference_mode(), self._dit_autocast():
for ref_frames in _WARMUP_REF_FRAMES:
@@ -283,6 +294,8 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
timestep=timestep,
cfg_strength=cfg,
new_request=True,
timesteps=grid[:-1],
step_index=0,
)
if self.device.type != "cpu":
torch.accelerator.synchronize(self.device)