[Perf][AuK] Precompute adaLN modulations per request and run the conv position embedding as one GEMM (#8328)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
1 parent
0e6791e9c4
commit
13c66da752
6 files changed
+329
-45
No files matched your search
@@ -61,7 +61,10 @@ def test_single_request_graph_wrapper_cpu_falls_back_to_eager(cfg_strength: floa
|
||||
|
||||
# The conditioning is prepared once per request and reused by every step.
|
||||
assert prepare_spy.call_count == 1
|
||||
assert prepare_spy.call_args.args[-1] is (cfg_strength >= 1e-5)
|
||||
# _prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg, timesteps)
|
||||
assert prepare_spy.call_args.args[6] is (cfg_strength >= 1e-5)
|
||||
# The time grid reaches prepare, so the adaLN modulations are computed once.
|
||||
assert prepare_spy.call_args.args[7].numel() == 2
|
||||
assert step_spy.call_count == 2
|
||||
capture_spy.assert_not_called()
|
||||
assert not wrapper._cache
|
||||
@@ -84,7 +87,8 @@ def test_graph_inputs_use_bounded_length_buckets() -> None:
|
||||
assert padded[4].shape == (1, 100, 4)
|
||||
assert padded[5].shape == (1, 100)
|
||||
assert [mask.sum().item() for mask in (padded[1], padded[3], padded[5])] == [65, 65, 51]
|
||||
assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False)
|
||||
assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False, 0)
|
||||
assert wrapper._key(padded[0], padded[2], padded[4], False, 32) == (96, 96, 100, False, 32)
|
||||
assert wrapper.max_graphs == 32
|
||||
|
||||
|
||||
@@ -153,18 +157,21 @@ def test_single_request_graph_replay_matches_eager_and_updates_inputs(cfg_streng
|
||||
inputs["ref"],
|
||||
inputs["ref_mask"],
|
||||
)
|
||||
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5)
|
||||
# sample_latents passes the grid, so the key carries its step count (two steps here).
|
||||
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5, 2)
|
||||
assert key in wrapper._cache
|
||||
|
||||
# The static context was refreshed for this request's conditioning.
|
||||
entry = wrapper._cache[key]
|
||||
fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5)
|
||||
grid = torch.tensor([0.0, 0.4], device="cuda")
|
||||
fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5, grid)
|
||||
for static, want in zip(entry.static_ctx.tensors(), fresh.tensors(), strict=True):
|
||||
if want is None:
|
||||
assert static is None
|
||||
else:
|
||||
torch.testing.assert_close(static, want)
|
||||
torch.testing.assert_close(entry.static_timestep, torch.tensor(0.4, device="cuda"))
|
||||
assert int(entry.static_step) == 1
|
||||
|
||||
assert len(wrapper._cache) == 1
|
||||
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""The AuK DiT fast paths keep the reference numerics.
|
||||
|
||||
Two paths are checked: the grouped conv position embedding as one batched
|
||||
GEMM, and the adaLN modulations precomputed for a whole time grid, eager and
|
||||
under the per-step CUDA graph.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.auk import auk_transformer as T
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, build_time_grid
|
||||
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_dit(device: str = "cpu") -> AuKTransformer:
|
||||
torch.manual_seed(5)
|
||||
return (
|
||||
AuKTransformer(
|
||||
dim=32,
|
||||
heads=2,
|
||||
dim_head=16,
|
||||
ff_mult=2,
|
||||
latent_dim=4,
|
||||
text_hidden_dim=8,
|
||||
num_layers=2,
|
||||
num_single_layers=2,
|
||||
)
|
||||
.eval()
|
||||
.to(device)
|
||||
)
|
||||
|
||||
|
||||
def _reference_conv_pos_embed(module: T.ConvPosEmbedding, x: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor:
|
||||
"""The channels-first conv stack as the reference implementation writes it."""
|
||||
keep = None if mask is None else mask.unsqueeze(1)
|
||||
x = x.transpose(1, 2)
|
||||
if keep is not None:
|
||||
x = x.masked_fill(~keep, 0.0)
|
||||
for layer in module.conv1d:
|
||||
x = layer(x)
|
||||
if keep is not None and isinstance(layer, torch.nn.Conv1d):
|
||||
x = x.masked_fill(~keep, 0.0)
|
||||
return x.transpose(1, 2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_grouped_conv_as_batched_gemm_matches_conv1d() -> None:
|
||||
torch.manual_seed(0)
|
||||
conv = torch.nn.Conv1d(64, 64, 31, groups=16, padding=15)
|
||||
x = torch.randn(2, 37, 64)
|
||||
expected = conv(x.transpose(1, 2)).transpose(1, 2)
|
||||
torch.testing.assert_close(T._grouped_conv1d_btc(x, conv), expected, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("masked", [False, True])
|
||||
def test_conv_pos_embedding_matches_the_channels_first_stack(masked: bool) -> None:
|
||||
torch.manual_seed(1)
|
||||
module = T.ConvPosEmbedding(64).eval()
|
||||
x = torch.randn(2, 37, 64)
|
||||
mask = None
|
||||
if masked:
|
||||
mask = torch.ones(2, 37, dtype=torch.bool)
|
||||
mask[1, 30:] = False
|
||||
torch.testing.assert_close(module(x, mask), _reference_conv_pos_embed(module, x, mask), atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("cfg_infer", [False, True])
|
||||
def test_modulation_table_matches_per_step_modulation(cfg_infer: bool) -> None:
|
||||
dit = _make_dit()
|
||||
text = torch.randn(1, 5, 8)
|
||||
ref = torch.randn(1, 3, 4)
|
||||
x = torch.randn(1, 6, 4)
|
||||
grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cpu")
|
||||
plain = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer)
|
||||
tabled = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer, timesteps=grid[:-1])
|
||||
assert tabled.modulation is not None and tabled.modulation.shape[0] == 4
|
||||
for i in range(4):
|
||||
expected = dit.step(x, grid[i], plain)
|
||||
torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=i), expected, atol=1e-5, rtol=1e-5)
|
||||
index = torch.tensor(i)
|
||||
torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=index), expected, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
def _padded_inputs(device: str) -> dict[str, torch.Tensor]:
|
||||
generator = torch.Generator(device="cpu").manual_seed(3)
|
||||
text = torch.randn(1, 7, 8, generator=generator)
|
||||
c_mask = torch.ones(1, 7, dtype=torch.bool)
|
||||
c_mask[:, 5:] = False
|
||||
text[:, 5:] = 0.0
|
||||
ref = torch.randn(1, 6, 4, generator=generator)
|
||||
ref_mask = torch.ones(1, 6, dtype=torch.bool)
|
||||
ref_mask[:, 4:] = False
|
||||
mask = torch.ones(1, 9, dtype=torch.bool)
|
||||
mask[:, 7:] = False
|
||||
inputs = {"text": text, "c_mask": c_mask, "ref": ref, "ref_mask": ref_mask, "mask": mask}
|
||||
return {name: value.to(device) for name, value in inputs.items()}
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA")
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
|
||||
def test_graph_with_time_grid_matches_eager_with_time_grid(cfg_strength: float) -> None:
|
||||
dit = _make_dit("cuda")
|
||||
inputs = _padded_inputs("cuda")
|
||||
grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cuda")
|
||||
x = torch.randn(1, 9, 4, device="cuda")
|
||||
wrapper = AuKCUDAGraphWrapper(dit)
|
||||
common = dict(
|
||||
text=inputs["text"],
|
||||
c_mask=inputs["c_mask"],
|
||||
ref=inputs["ref"],
|
||||
ref_mask=inputs["ref_mask"],
|
||||
cfg_strength=cfg_strength,
|
||||
timesteps=grid[:-1],
|
||||
)
|
||||
for i in range(4):
|
||||
graph = wrapper(x=x, timestep=grid[i], new_request=i == 0, step_index=i, **common)
|
||||
ctx = dit.prepare(
|
||||
inputs["text"],
|
||||
target_len=9,
|
||||
c_mask=inputs["c_mask"],
|
||||
ref=inputs["ref"],
|
||||
ref_mask=inputs["ref_mask"],
|
||||
cfg_infer=cfg_strength >= 1e-5,
|
||||
timesteps=grid[:-1],
|
||||
)
|
||||
eager = wrapper._step(x, grid[i], ctx, cfg_strength, i)
|
||||
torch.testing.assert_close(graph, eager, atol=1e-4, rtol=1e-4)
|
||||
# One graph serves every step of the grid.
|
||||
assert len(wrapper._cache) == 1
|
||||
assert next(iter(wrapper._cache))[-1] == 4
|
||||
@@ -127,7 +127,9 @@ def test_hoisted_euler_loop_matches_per_step_forward(cfg_strength: float) -> Non
|
||||
v = dit(x, time=timesteps[i], **inputs)
|
||||
x = x + (timesteps[i + 1] - timesteps[i]) * v
|
||||
|
||||
torch.testing.assert_close(got, x, rtol=0, atol=0)
|
||||
# sample_latents precomputes the adaLN modulations of the whole grid in one
|
||||
# GEMM per layer, whose rows round differently from one-row GEMMs in fp32.
|
||||
torch.testing.assert_close(got, x, rtol=0, atol=1e-6)
|
||||
assert dit.text_cond is None and dit.text_uncond is None
|
||||
|
||||
|
||||
|
||||
@@ -153,6 +153,33 @@ class TimeEmbedding(nn.Module):
|
||||
return self.time_mlp(hidden.to(self.time_mlp[0].weight.dtype))
|
||||
|
||||
|
||||
def _grouped_conv1d_btc(x: torch.Tensor, conv: nn.Conv1d) -> torch.Tensor:
|
||||
"""``conv`` applied to channels-last ``x`` ``[B, T, C]``, returning ``[B, T, C]``.
|
||||
|
||||
A same-padded grouped convolution written as one im2col copy and one
|
||||
batched GEMM over the groups. cuDNN runs a grouped conv of this shape as
|
||||
one kernel per group plus two layout transposes, which at AuK's sizes
|
||||
(1536 channels in 16 groups, kernel 31, a few hundred frames) costs about
|
||||
ten times the arithmetic.
|
||||
"""
|
||||
batch, frames, channels = x.shape
|
||||
groups = conv.groups
|
||||
kernel = conv.kernel_size[0]
|
||||
per_group = channels // groups
|
||||
pad = conv.padding[0]
|
||||
padded = F.pad(x, (0, 0, pad, pad))
|
||||
# unfold gives a [B, T, C, K] view; one copy lays it out as [B, T, G, C/G * K].
|
||||
cols = padded.unfold(1, kernel, 1).reshape(batch * frames, groups, per_group * kernel)
|
||||
# conv.weight [C_out, C/G, K] -> [G, C/G * K, C_out/G]; the reduction order matches the columns.
|
||||
weight = conv.weight.reshape(groups, channels // groups, per_group * kernel).transpose(1, 2).to(cols.dtype)
|
||||
# [G, B*T, C/G*K] @ [G, C/G*K, C_out/G] -> [G, B*T, C_out/G]
|
||||
out = torch.bmm(cols.transpose(0, 1), weight)
|
||||
out = out.transpose(0, 1).reshape(batch, frames, channels)
|
||||
if conv.bias is not None:
|
||||
out = out + conv.bias.to(out.dtype)
|
||||
return out
|
||||
|
||||
|
||||
class ConvPosEmbedding(nn.Module):
|
||||
"""Depthwise-grouped conv stack that adds local position information."""
|
||||
|
||||
@@ -169,17 +196,19 @@ class ConvPosEmbedding(nn.Module):
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
|
||||
keep = None if mask is None else mask.unsqueeze(1)
|
||||
x = x.transpose(1, 2)
|
||||
keep = None if mask is None else mask.unsqueeze(-1)
|
||||
if keep is not None:
|
||||
x = x.masked_fill(~keep, 0.0)
|
||||
for layer in self.conv1d:
|
||||
x = layer(x)
|
||||
# Re-zero padding after each convolution so it cannot bleed into
|
||||
# valid frames through the kernel window.
|
||||
if keep is not None and isinstance(layer, nn.Conv1d):
|
||||
x = x.masked_fill(~keep, 0.0)
|
||||
return x.transpose(1, 2)
|
||||
if isinstance(layer, nn.Conv1d):
|
||||
x = _grouped_conv1d_btc(x, layer)
|
||||
# Re-zero padding after each convolution so it cannot bleed into
|
||||
# valid frames through the kernel window.
|
||||
if keep is not None:
|
||||
x = x.masked_fill(~keep, 0.0)
|
||||
else:
|
||||
x = layer(x)
|
||||
return x
|
||||
|
||||
|
||||
class AudioEmbedding(nn.Module):
|
||||
@@ -205,9 +234,12 @@ class AdaLayerNorm(nn.Module):
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, emb: torch.Tensor
|
||||
self, x: torch.Tensor, emb: torch.Tensor | None, mod: torch.Tensor | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.linear(self.silu(emb)).chunk(6, dim=1)
|
||||
"""``mod`` is the precomputed ``linear(silu(emb))``; ``emb`` is then unused."""
|
||||
if mod is None:
|
||||
mod = self.linear(self.silu(emb))
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=1)
|
||||
x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
|
||||
return x, gate_msa, shift_mlp, scale_mlp, gate_mlp
|
||||
|
||||
@@ -221,8 +253,11 @@ class AdaLayerNormFinal(nn.Module):
|
||||
self.linear = nn.Linear(dim, dim * 2)
|
||||
self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
|
||||
def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor:
|
||||
scale, shift = self.linear(self.silu(emb)).chunk(2, dim=1)
|
||||
def forward(self, x: torch.Tensor, emb: torch.Tensor | None, mod: torch.Tensor | None = None) -> torch.Tensor:
|
||||
"""``mod`` is the precomputed ``linear(silu(emb))``; ``emb`` is then unused."""
|
||||
if mod is None:
|
||||
mod = self.linear(self.silu(emb))
|
||||
scale, shift = mod.chunk(2, dim=1)
|
||||
return self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
|
||||
|
||||
|
||||
@@ -383,9 +418,11 @@ class DoubleBlock(nn.Module):
|
||||
c_rope: RopeCache,
|
||||
c_mask: torch.Tensor | None,
|
||||
bias: torch.Tensor | None = None,
|
||||
mod_c: torch.Tensor | None = None,
|
||||
mod_x: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t)
|
||||
norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t)
|
||||
norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t, mod_c)
|
||||
norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t, mod_x)
|
||||
|
||||
x_attn, c_attn = self.attn(x=norm_x, c=norm_c, mask=mask, rope=rope, c_rope=c_rope, c_mask=c_mask, bias=bias)
|
||||
|
||||
@@ -416,8 +453,9 @@ class SingleBlock(nn.Module):
|
||||
mask: torch.Tensor | None,
|
||||
rope: RopeCache,
|
||||
bias: torch.Tensor | None = None,
|
||||
mod: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t)
|
||||
norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t, mod)
|
||||
x = x + gate_msa[:, None] * self.attn(x=norm, mask=mask, rope=rope, bias=bias)
|
||||
norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
|
||||
return x + gate_mlp[:, None] * self.ff(norm)
|
||||
@@ -491,6 +529,28 @@ class AuKTransformer(nn.Module):
|
||||
|
||||
self.norm_out = AdaLayerNormFinal(dim)
|
||||
self.proj_out = nn.Linear(dim, latent_dim)
|
||||
# Widths of the adaLN modulations in the order modulation_table() lays them out.
|
||||
self._modulation_widths = [module.linear.out_features for module in self._adaln_modules()]
|
||||
|
||||
def _adaln_modules(self) -> list[nn.Module]:
|
||||
"""Every timestep-conditioned norm, in the order :meth:`step` consumes them."""
|
||||
modules: list[nn.Module] = []
|
||||
for block in self.transformer_blocks:
|
||||
modules += [block.attn_norm_c, block.attn_norm_x]
|
||||
modules += [block.attn_norm for block in self.single_transformer_blocks]
|
||||
modules.append(self.norm_out)
|
||||
return modules
|
||||
|
||||
def modulation_table(self, timesteps: torch.Tensor) -> torch.Tensor:
|
||||
"""adaLN modulations for every timestep, ``[len(timesteps), sum of widths]``.
|
||||
|
||||
The modulations depend only on the timestep, and a request's time grid
|
||||
is known before its first step. Computing them for the whole grid at
|
||||
once reads each modulation weight once per request instead of once per
|
||||
step, which at batch 1 is what those matrix-vector products cost.
|
||||
"""
|
||||
emb = F.silu(self.time_embed(timesteps.reshape(-1).float()))
|
||||
return torch.cat([module.linear(emb) for module in self._adaln_modules()], dim=1)
|
||||
|
||||
def project_text(self, text: torch.Tensor) -> torch.Tensor:
|
||||
"""Project LLM hidden states ``[B, nt, text_hidden_dim]`` to model width."""
|
||||
@@ -520,6 +580,7 @@ class AuKTransformer(nn.Module):
|
||||
drop_text: bool = False,
|
||||
cfg_infer: bool = False,
|
||||
cache: bool = False,
|
||||
timesteps: torch.Tensor | None = None,
|
||||
) -> "AuKStepContext":
|
||||
"""Everything a denoise step needs that does not depend on ``x`` or the time.
|
||||
|
||||
@@ -528,6 +589,9 @@ class AuKTransformer(nn.Module):
|
||||
prompt embedding, the padding masks and biases and the rotary tables
|
||||
are not recomputed on every step. Arguments are those of
|
||||
:meth:`forward`; ``target_len`` is the target frame count ``n``.
|
||||
``timesteps`` are the times the steps will run at (the grid without
|
||||
its end point); when given, their adaLN modulations are precomputed
|
||||
and :meth:`step` selects them by ``step_index``.
|
||||
"""
|
||||
batch = text.shape[0]
|
||||
if c_mask is None:
|
||||
@@ -593,25 +657,44 @@ class AuKTransformer(nn.Module):
|
||||
rope_text=_rope_cos_sin(self.rotary_embed(text_len, c_mask)),
|
||||
rope_single=_rope_cos_sin(self.rotary_embed(text_len + audio_len, single_mask)),
|
||||
branches=branches,
|
||||
modulation=None if timesteps is None else self.modulation_table(timesteps),
|
||||
)
|
||||
|
||||
def step(self, x: torch.Tensor, time: torch.Tensor, ctx: "AuKStepContext") -> torch.Tensor:
|
||||
def step(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
time: torch.Tensor,
|
||||
ctx: "AuKStepContext",
|
||||
step_index: torch.Tensor | int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Predict the velocity of ``x`` at ``time`` under a :meth:`prepare` context.
|
||||
|
||||
The target embedding does not depend on the CFG branch, so it is
|
||||
computed once and shared by both branches.
|
||||
computed once and shared by both branches. When the context carries a
|
||||
modulation table, ``step_index`` selects this step's row of it and
|
||||
``time`` is unused; the row broadcasts over the CFG branches.
|
||||
"""
|
||||
if time.ndim == 0:
|
||||
time = time.repeat(x.shape[0])
|
||||
t = self.time_embed(time)
|
||||
target = self.audio_embed(x, mask=ctx.target_mask)
|
||||
if ctx.modulation is not None and step_index is not None:
|
||||
if isinstance(step_index, int):
|
||||
row = ctx.modulation[step_index : step_index + 1]
|
||||
else:
|
||||
row = ctx.modulation.index_select(0, step_index.reshape(1))
|
||||
mods = list(torch.split(row, self._modulation_widths, dim=1))
|
||||
t = None
|
||||
else:
|
||||
if time.ndim == 0:
|
||||
time = time.repeat(x.shape[0])
|
||||
t = self.time_embed(time)
|
||||
if ctx.branches == 2:
|
||||
t = torch.cat((t, t), dim=0)
|
||||
mods = [None] * len(self._modulation_widths)
|
||||
if ctx.branches == 2:
|
||||
t = torch.cat((t, t), dim=0)
|
||||
target = torch.cat((target, target), dim=0)
|
||||
audio = target if ctx.prompt is None else torch.cat([ctx.prompt, target], dim=1)
|
||||
c = ctx.c
|
||||
|
||||
for block in self.transformer_blocks:
|
||||
for i, block in enumerate(self.transformer_blocks):
|
||||
c, audio = block(
|
||||
audio,
|
||||
c,
|
||||
@@ -621,14 +704,24 @@ class AuKTransformer(nn.Module):
|
||||
c_rope=ctx.rope_text,
|
||||
c_mask=ctx.c_mask,
|
||||
bias=ctx.joint_bias,
|
||||
mod_c=mods[2 * i],
|
||||
mod_x=mods[2 * i + 1],
|
||||
)
|
||||
|
||||
h = torch.cat([c, audio], dim=1)
|
||||
for block in self.single_transformer_blocks:
|
||||
h = block(h, t, mask=ctx.single_mask, rope=ctx.rope_single, bias=ctx.single_bias)
|
||||
offset = 2 * len(self.transformer_blocks)
|
||||
for i, block in enumerate(self.single_transformer_blocks):
|
||||
h = block(
|
||||
h,
|
||||
t,
|
||||
mask=ctx.single_mask,
|
||||
rope=ctx.rope_single,
|
||||
bias=ctx.single_bias,
|
||||
mod=mods[offset + i],
|
||||
)
|
||||
|
||||
h = h[:, h.shape[1] - x.shape[1] :]
|
||||
return self.proj_out(self.norm_out(h, t))
|
||||
return self.proj_out(self.norm_out(h, t, mods[-1]))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -706,6 +799,8 @@ class AuKStepContext:
|
||||
rope_text: RopeCache
|
||||
rope_single: RopeCache
|
||||
branches: int
|
||||
#: adaLN modulations per step, ``[steps, widths]``, when the time grid was known.
|
||||
modulation: torch.Tensor | None = None
|
||||
|
||||
def tensors(self) -> list[torch.Tensor | None]:
|
||||
"""Every tensor field in a fixed order, rope pairs flattened."""
|
||||
@@ -721,6 +816,7 @@ class AuKStepContext:
|
||||
*self.rope_audio,
|
||||
*self.rope_text,
|
||||
*self.rope_single,
|
||||
self.modulation,
|
||||
]
|
||||
|
||||
def copy_(self, other: "AuKStepContext") -> None:
|
||||
@@ -752,6 +848,7 @@ class AuKStepContext:
|
||||
rope_text=(self.rope_text[0].clone(), self.rope_text[1].clone()),
|
||||
rope_single=(self.rope_single[0].clone(), self.rope_single[1].clone()),
|
||||
branches=self.branches,
|
||||
modulation=_clone(self.modulation),
|
||||
)
|
||||
|
||||
|
||||
@@ -881,6 +978,8 @@ def sample_latents(
|
||||
timestep=t[i],
|
||||
cfg_strength=cfg_strength,
|
||||
new_request=i == 0,
|
||||
timesteps=t[:-1],
|
||||
step_index=i,
|
||||
)
|
||||
x = x + (t[i + 1] - t[i]) * velocity
|
||||
finally:
|
||||
@@ -922,9 +1021,10 @@ def _sample_latents(
|
||||
ref_mask=ref_mask,
|
||||
cfg_infer=guided,
|
||||
cache=guided,
|
||||
timesteps=timesteps[:-1],
|
||||
)
|
||||
for i in range(timesteps.shape[0] - 1):
|
||||
v = dit.step(x, timesteps[i], ctx)
|
||||
v = dit.step(x, timesteps[i], ctx, step_index=i)
|
||||
if guided:
|
||||
v_cond, v_uncond = v.chunk(2, dim=0)
|
||||
v = v_cond + (v_cond - v_uncond) * cfg_strength
|
||||
|
||||
@@ -24,16 +24,19 @@ class _GraphEntry:
|
||||
static_ctx: AuKStepContext
|
||||
static_timestep: torch.Tensor
|
||||
static_cfg: torch.Tensor | None
|
||||
static_step: torch.Tensor | None
|
||||
static_out: torch.Tensor
|
||||
|
||||
|
||||
class AuKCUDAGraphWrapper:
|
||||
"""Replay one DiT denoise step and leave Euler scheduling to the caller.
|
||||
|
||||
The graph is keyed by the target, text and reference sequence lengths plus
|
||||
whether the CFG branch is enabled. Timestep and the CFG strength are
|
||||
The graph is keyed by the target, text and reference sequence lengths,
|
||||
whether the CFG branch is enabled and, when the caller passes the time
|
||||
grid, its length. Timestep (or the step index), and the CFG strength are
|
||||
mutable scalar buffers, so all Euler steps and CFG values within one path
|
||||
reuse one graph.
|
||||
reuse one graph. With the time grid, the adaLN modulations of every step
|
||||
are part of the per-request context and the graph selects its row.
|
||||
|
||||
Only :meth:`AuKTransformer.step` is captured. The per-request context
|
||||
(text projection, reference embedding, padding biases, rotary tables) is
|
||||
@@ -85,8 +88,9 @@ class AuKCUDAGraphWrapper:
|
||||
text: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
uses_cfg: bool,
|
||||
) -> tuple[int, int, int, bool]:
|
||||
return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg)
|
||||
steps: int = 0,
|
||||
) -> tuple[int, int, int, bool, int]:
|
||||
return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg, steps)
|
||||
|
||||
def _retire_graph_generation_if_full(self) -> None:
|
||||
"""Retire all graphs together so none outlive shared workspaces."""
|
||||
@@ -103,6 +107,7 @@ class AuKCUDAGraphWrapper:
|
||||
ref: torch.Tensor,
|
||||
ref_mask: torch.Tensor,
|
||||
uses_cfg: bool,
|
||||
timesteps: torch.Tensor | None = None,
|
||||
) -> AuKStepContext:
|
||||
return self.dit.prepare(
|
||||
text,
|
||||
@@ -112,6 +117,7 @@ class AuKCUDAGraphWrapper:
|
||||
ref=ref,
|
||||
ref_mask=ref_mask,
|
||||
cfg_infer=uses_cfg,
|
||||
timesteps=timesteps,
|
||||
)
|
||||
|
||||
def _step(
|
||||
@@ -120,9 +126,10 @@ class AuKCUDAGraphWrapper:
|
||||
timestep: torch.Tensor,
|
||||
ctx: AuKStepContext,
|
||||
cfg_strength: torch.Tensor | float,
|
||||
step_index: torch.Tensor | int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""One guided (or unguided) velocity; the CFG combine is part of the graph."""
|
||||
velocity = self.dit.step(x, timestep, ctx)
|
||||
velocity = self.dit.step(x, timestep, ctx, step_index=step_index)
|
||||
if ctx.branches == 1:
|
||||
return velocity
|
||||
conditional, unconditional = velocity.chunk(2, dim=0)
|
||||
@@ -140,6 +147,8 @@ class AuKCUDAGraphWrapper:
|
||||
timestep: torch.Tensor,
|
||||
cfg_strength: float,
|
||||
new_request: bool = True,
|
||||
timesteps: torch.Tensor | None = None,
|
||||
step_index: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Velocity of ``x`` at ``timestep``.
|
||||
|
||||
@@ -147,26 +156,33 @@ class AuKCUDAGraphWrapper:
|
||||
(``text``, ``c_mask``, ``ref``, ``ref_mask``) is then prepared anew; the
|
||||
later steps of the request pass ``False`` and reuse it. The default
|
||||
prepares on every call, which is always correct.
|
||||
|
||||
``timesteps`` (the times every step of the request runs at) and
|
||||
``step_index`` (this step's position in them) let the adaLN
|
||||
modulations be computed once per request instead of once per step.
|
||||
"""
|
||||
uses_cfg = cfg_strength >= 1e-5
|
||||
if timesteps is None or step_index is None:
|
||||
timesteps, step_index = None, None
|
||||
if not self.enabled or x.device.type != "cuda" or torch.cuda.is_current_stream_capturing():
|
||||
if new_request or self._eager_ctx is None:
|
||||
self._eager_ctx = self._prepare(x, None, text, c_mask, ref, ref_mask, uses_cfg)
|
||||
return self._step(x, timestep, self._eager_ctx, cfg_strength)
|
||||
self._eager_ctx = self._prepare(x, None, text, c_mask, ref, ref_mask, uses_cfg, timesteps)
|
||||
return self._step(x, timestep, self._eager_ctx, cfg_strength, step_index)
|
||||
|
||||
target_frames = x.shape[1]
|
||||
x, x_mask, text, c_mask, ref, ref_mask = self._bucket_inputs(x, text, c_mask, ref, ref_mask)
|
||||
key = self._key(x, text, ref, uses_cfg)
|
||||
steps = 0 if timesteps is None else int(timesteps.numel())
|
||||
key = self._key(x, text, ref, uses_cfg, steps)
|
||||
entry = self._cache.get(key)
|
||||
ctx = None
|
||||
if entry is None or new_request or key != self._active_key:
|
||||
ctx = self._prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg)
|
||||
ctx = self._prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg, timesteps)
|
||||
if entry is None:
|
||||
# Same Retirement as #6587, details discussed in #7469
|
||||
self._retire_graph_generation_if_full()
|
||||
try:
|
||||
assert ctx is not None
|
||||
entry = self._capture(x, ctx, timestep, cfg_strength=cfg_strength)
|
||||
entry = self._capture(x, ctx, timestep, cfg_strength=cfg_strength, step_index=step_index)
|
||||
except Exception:
|
||||
self._cache.clear()
|
||||
self._active_key = None
|
||||
@@ -186,6 +202,8 @@ class AuKCUDAGraphWrapper:
|
||||
|
||||
entry.static_x.copy_(x)
|
||||
entry.static_timestep.copy_(timestep)
|
||||
if entry.static_step is not None:
|
||||
entry.static_step.fill_(step_index)
|
||||
if entry.static_cfg is not None:
|
||||
entry.static_cfg.fill_(cfg_strength)
|
||||
entry.graph.replay()
|
||||
@@ -198,10 +216,14 @@ class AuKCUDAGraphWrapper:
|
||||
timestep: torch.Tensor,
|
||||
*,
|
||||
cfg_strength: float,
|
||||
step_index: int | None = None,
|
||||
) -> _GraphEntry:
|
||||
static_x = x.clone()
|
||||
static_ctx = ctx.clone()
|
||||
static_timestep = timestep.clone()
|
||||
static_step = None
|
||||
if static_ctx.modulation is not None and step_index is not None:
|
||||
static_step = torch.full((), step_index, device=static_x.device, dtype=torch.long)
|
||||
static_cfg = None
|
||||
if ctx.branches == 2:
|
||||
static_cfg = torch.empty((), device=static_x.device, dtype=torch.float32)
|
||||
@@ -209,13 +231,13 @@ class AuKCUDAGraphWrapper:
|
||||
scale = static_cfg if static_cfg is not None else cfg_strength
|
||||
# Warm-up runs any lazy torch.compile outside the capture.
|
||||
for _ in range(3):
|
||||
self._step(static_x, static_timestep, static_ctx, scale)
|
||||
self._step(static_x, static_timestep, static_ctx, scale, static_step)
|
||||
if self._pool_handle is None:
|
||||
self._pool_handle = torch.cuda.graph_pool_handle()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, pool=self._pool_handle):
|
||||
# The CFG scalar stays a mutable buffer, so cfg=2 and cfg=3 share this graph.
|
||||
static_out = self._step(static_x, static_timestep, static_ctx, scale)
|
||||
static_out = self._step(static_x, static_timestep, static_ctx, scale, static_step)
|
||||
|
||||
logger.info(
|
||||
"Captured AuK DiT single-step CUDA graph: target_frames=%d text_tokens=%d ref_frames=%d cfg=%s",
|
||||
@@ -230,6 +252,7 @@ class AuKCUDAGraphWrapper:
|
||||
static_ctx=static_ctx,
|
||||
static_timestep=static_timestep,
|
||||
static_cfg=static_cfg,
|
||||
static_step=static_step,
|
||||
static_out=static_out,
|
||||
)
|
||||
|
||||
|
||||
@@ -27,7 +27,12 @@ from vllm.logger import init_logger
|
||||
from vllm_omni.diffusion.compile import regionally_compile
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import (
|
||||
AuKTransformer,
|
||||
build_time_grid,
|
||||
dit_state_dict,
|
||||
sample_latents,
|
||||
)
|
||||
from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE
|
||||
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
|
||||
from vllm_omni.diffusion.models.auk.vae_cudagraph import AuKVAEDecodeGraph
|
||||
@@ -267,6 +272,12 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
|
||||
text = torch.zeros(1, _WARMUP_TEXT_TOKENS, self.text_hidden_dim, device=self.device, dtype=self.dtype)
|
||||
c_mask = torch.ones(text.shape[:2], dtype=torch.bool, device=self.device)
|
||||
timestep = torch.zeros((), device=self.device, dtype=torch.float32)
|
||||
# The graphs are keyed by the number of steps, so warm the variant's default grid.
|
||||
if self.is_flash:
|
||||
grid = torch.tensor(self.flash_t_grid, device=self.device, dtype=torch.float32)
|
||||
else:
|
||||
sway = None if self.default_sway is None else float(self.default_sway)
|
||||
grid = build_time_grid(nfe=self.default_nfe, sway_sampling_coef=sway, t_grid=None, device=self.device)
|
||||
start = time.perf_counter()
|
||||
with torch.inference_mode(), self._dit_autocast():
|
||||
for ref_frames in _WARMUP_REF_FRAMES:
|
||||
@@ -283,6 +294,8 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp
|
||||
timestep=timestep,
|
||||
cfg_strength=cfg,
|
||||
new_request=True,
|
||||
timesteps=grid[:-1],
|
||||
step_index=0,
|
||||
)
|
||||
if self.device.type != "cpu":
|
||||
torch.accelerator.synchronize(self.device)
|
||||
|
||||
Reference in new issue
Block a user