[Perf][AuK] Precompute adaLN modulations per request and run the conv position embedding as one GEMM (#8328)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-10-02 08:12:29 -07:00
1 parent 0e6791e9c4
commit 13c66da752
6 files changed
+329 -45

No files matched your search

@@ -61,7 +61,10 @@ def test_single_request_graph_wrapper_cpu_falls_back_to_eager(cfg_strength: floa
# The conditioning is prepared once per request and reused by every step.
assert prepare_spy.call_count == 1
assert prepare_spy.call_args.args[-1] is (cfg_strength >= 1e-5)
# _prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg, timesteps)
assert prepare_spy.call_args.args[6] is (cfg_strength >= 1e-5)
# The time grid reaches prepare, so the adaLN modulations are computed once.
assert prepare_spy.call_args.args[7].numel() == 2
assert step_spy.call_count == 2
capture_spy.assert_not_called()
assert not wrapper._cache
@@ -84,7 +87,8 @@ def test_graph_inputs_use_bounded_length_buckets() -> None:
assert padded[4].shape == (1, 100, 4)
assert padded[5].shape == (1, 100)
assert [mask.sum().item() for mask in (padded[1], padded[3], padded[5])] == [65, 65, 51]
assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False)
assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False, 0)
assert wrapper._key(padded[0], padded[2], padded[4], False, 32) == (96, 96, 100, False, 32)
assert wrapper.max_graphs == 32
@@ -153,18 +157,21 @@ def test_single_request_graph_replay_matches_eager_and_updates_inputs(cfg_streng
inputs["ref"],
inputs["ref_mask"],
)
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5)
# sample_latents passes the grid, so the key carries its step count (two steps here).
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5, 2)
assert key in wrapper._cache
# The static context was refreshed for this request's conditioning.
entry = wrapper._cache[key]
fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5)
grid = torch.tensor([0.0, 0.4], device="cuda")
fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5, grid)
for static, want in zip(entry.static_ctx.tensors(), fresh.tensors(), strict=True):
if want is None:
assert static is None
else:
torch.testing.assert_close(static, want)
torch.testing.assert_close(entry.static_timestep, torch.tensor(0.4, device="cuda"))
assert int(entry.static_step) == 1
assert len(wrapper._cache) == 1
@@ -0,0 +1,139 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""The AuK DiT fast paths keep the reference numerics.
Two paths are checked: the grouped conv position embedding as one batched
GEMM, and the adaLN modulations precomputed for a whole time grid, eager and
under the per-step CUDA graph.
"""
import pytest
import torch
from vllm_omni.diffusion.models.auk import auk_transformer as T
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, build_time_grid
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_dit(device: str = "cpu") -> AuKTransformer:
torch.manual_seed(5)
return (
AuKTransformer(
dim=32,
heads=2,
dim_head=16,
ff_mult=2,
latent_dim=4,
text_hidden_dim=8,
num_layers=2,
num_single_layers=2,
)
.eval()
.to(device)
)
def _reference_conv_pos_embed(module: T.ConvPosEmbedding, x: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor:
"""The channels-first conv stack as the reference implementation writes it."""
keep = None if mask is None else mask.unsqueeze(1)
x = x.transpose(1, 2)
if keep is not None:
x = x.masked_fill(~keep, 0.0)
for layer in module.conv1d:
x = layer(x)
if keep is not None and isinstance(layer, torch.nn.Conv1d):
x = x.masked_fill(~keep, 0.0)
return x.transpose(1, 2)
@torch.inference_mode()
def test_grouped_conv_as_batched_gemm_matches_conv1d() -> None:
torch.manual_seed(0)
conv = torch.nn.Conv1d(64, 64, 31, groups=16, padding=15)
x = torch.randn(2, 37, 64)
expected = conv(x.transpose(1, 2)).transpose(1, 2)
torch.testing.assert_close(T._grouped_conv1d_btc(x, conv), expected, atol=1e-5, rtol=1e-5)
@torch.inference_mode()
@pytest.mark.parametrize("masked", [False, True])
def test_conv_pos_embedding_matches_the_channels_first_stack(masked: bool) -> None:
torch.manual_seed(1)
module = T.ConvPosEmbedding(64).eval()
x = torch.randn(2, 37, 64)
mask = None
if masked:
mask = torch.ones(2, 37, dtype=torch.bool)
mask[1, 30:] = False
torch.testing.assert_close(module(x, mask), _reference_conv_pos_embed(module, x, mask), atol=1e-5, rtol=1e-5)
@torch.inference_mode()
@pytest.mark.parametrize("cfg_infer", [False, True])
def test_modulation_table_matches_per_step_modulation(cfg_infer: bool) -> None:
dit = _make_dit()
text = torch.randn(1, 5, 8)
ref = torch.randn(1, 3, 4)
x = torch.randn(1, 6, 4)
grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cpu")
plain = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer)
tabled = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer, timesteps=grid[:-1])
assert tabled.modulation is not None and tabled.modulation.shape[0] == 4
for i in range(4):
expected = dit.step(x, grid[i], plain)
torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=i), expected, atol=1e-5, rtol=1e-5)
index = torch.tensor(i)
torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=index), expected, atol=1e-5, rtol=1e-5)
def _padded_inputs(device: str) -> dict[str, torch.Tensor]:
generator = torch.Generator(device="cpu").manual_seed(3)
text = torch.randn(1, 7, 8, generator=generator)
c_mask = torch.ones(1, 7, dtype=torch.bool)
c_mask[:, 5:] = False
text[:, 5:] = 0.0
ref = torch.randn(1, 6, 4, generator=generator)
ref_mask = torch.ones(1, 6, dtype=torch.bool)
ref_mask[:, 4:] = False
mask = torch.ones(1, 9, dtype=torch.bool)
mask[:, 7:] = False
inputs = {"text": text, "c_mask": c_mask, "ref": ref, "ref_mask": ref_mask, "mask": mask}
return {name: value.to(device) for name, value in inputs.items()}
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA")
@torch.inference_mode()
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
def test_graph_with_time_grid_matches_eager_with_time_grid(cfg_strength: float) -> None:
dit = _make_dit("cuda")
inputs = _padded_inputs("cuda")
grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cuda")
x = torch.randn(1, 9, 4, device="cuda")
wrapper = AuKCUDAGraphWrapper(dit)
common = dict(
text=inputs["text"],
c_mask=inputs["c_mask"],
ref=inputs["ref"],
ref_mask=inputs["ref_mask"],
cfg_strength=cfg_strength,
timesteps=grid[:-1],
)
for i in range(4):
graph = wrapper(x=x, timestep=grid[i], new_request=i == 0, step_index=i, **common)
ctx = dit.prepare(
inputs["text"],
target_len=9,
c_mask=inputs["c_mask"],
ref=inputs["ref"],
ref_mask=inputs["ref_mask"],
cfg_infer=cfg_strength >= 1e-5,
timesteps=grid[:-1],
)
eager = wrapper._step(x, grid[i], ctx, cfg_strength, i)
torch.testing.assert_close(graph, eager, atol=1e-4, rtol=1e-4)
# One graph serves every step of the grid.
assert len(wrapper._cache) == 1
assert next(iter(wrapper._cache))[-1] == 4
@@ -127,7 +127,9 @@ def test_hoisted_euler_loop_matches_per_step_forward(cfg_strength: float) -> Non
v = dit(x, time=timesteps[i], **inputs)
x = x + (timesteps[i + 1] - timesteps[i]) * v
torch.testing.assert_close(got, x, rtol=0, atol=0)
# sample_latents precomputes the adaLN modulations of the whole grid in one
# GEMM per layer, whose rows round differently from one-row GEMMs in fp32.
torch.testing.assert_close(got, x, rtol=0, atol=1e-6)
assert dit.text_cond is None and dit.text_uncond is None