diff --git a/tests/diffusion/models/auk/test_auk_cudagraph.py b/tests/diffusion/models/auk/test_auk_cudagraph.py index 663cdf8df..c140b8d6c 100644 --- a/tests/diffusion/models/auk/test_auk_cudagraph.py +++ b/tests/diffusion/models/auk/test_auk_cudagraph.py @@ -61,7 +61,10 @@ def test_single_request_graph_wrapper_cpu_falls_back_to_eager(cfg_strength: floa # The conditioning is prepared once per request and reused by every step. assert prepare_spy.call_count == 1 - assert prepare_spy.call_args.args[-1] is (cfg_strength >= 1e-5) + # _prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg, timesteps) + assert prepare_spy.call_args.args[6] is (cfg_strength >= 1e-5) + # The time grid reaches prepare, so the adaLN modulations are computed once. + assert prepare_spy.call_args.args[7].numel() == 2 assert step_spy.call_count == 2 capture_spy.assert_not_called() assert not wrapper._cache @@ -84,7 +87,8 @@ def test_graph_inputs_use_bounded_length_buckets() -> None: assert padded[4].shape == (1, 100, 4) assert padded[5].shape == (1, 100) assert [mask.sum().item() for mask in (padded[1], padded[3], padded[5])] == [65, 65, 51] - assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False) + assert wrapper._key(padded[0], padded[2], padded[4], False) == (96, 96, 100, False, 0) + assert wrapper._key(padded[0], padded[2], padded[4], False, 32) == (96, 96, 100, False, 32) assert wrapper.max_graphs == 32 @@ -153,18 +157,21 @@ def test_single_request_graph_replay_matches_eager_and_updates_inputs(cfg_streng inputs["ref"], inputs["ref_mask"], ) - key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5) + # sample_latents passes the grid, so the key carries its step count (two steps here). + key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5, 2) assert key in wrapper._cache # The static context was refreshed for this request's conditioning. entry = wrapper._cache[key] - fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5) + grid = torch.tensor([0.0, 0.4], device="cuda") + fresh = wrapper._prepare(*bucketed, cfg_strength >= 1e-5, grid) for static, want in zip(entry.static_ctx.tensors(), fresh.tensors(), strict=True): if want is None: assert static is None else: torch.testing.assert_close(static, want) torch.testing.assert_close(entry.static_timestep, torch.tensor(0.4, device="cuda")) + assert int(entry.static_step) == 1 assert len(wrapper._cache) == 1 diff --git a/tests/diffusion/models/auk/test_auk_dit_fastpaths.py b/tests/diffusion/models/auk/test_auk_dit_fastpaths.py new file mode 100644 index 000000000..97d97e619 --- /dev/null +++ b/tests/diffusion/models/auk/test_auk_dit_fastpaths.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""The AuK DiT fast paths keep the reference numerics. + +Two paths are checked: the grouped conv position embedding as one batched +GEMM, and the adaLN modulations precomputed for a whole time grid, eager and +under the per-step CUDA graph. +""" + +import pytest +import torch + +from vllm_omni.diffusion.models.auk import auk_transformer as T +from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, build_time_grid +from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_dit(device: str = "cpu") -> AuKTransformer: + torch.manual_seed(5) + return ( + AuKTransformer( + dim=32, + heads=2, + dim_head=16, + ff_mult=2, + latent_dim=4, + text_hidden_dim=8, + num_layers=2, + num_single_layers=2, + ) + .eval() + .to(device) + ) + + +def _reference_conv_pos_embed(module: T.ConvPosEmbedding, x: torch.Tensor, mask: torch.Tensor | None) -> torch.Tensor: + """The channels-first conv stack as the reference implementation writes it.""" + keep = None if mask is None else mask.unsqueeze(1) + x = x.transpose(1, 2) + if keep is not None: + x = x.masked_fill(~keep, 0.0) + for layer in module.conv1d: + x = layer(x) + if keep is not None and isinstance(layer, torch.nn.Conv1d): + x = x.masked_fill(~keep, 0.0) + return x.transpose(1, 2) + + +@torch.inference_mode() +def test_grouped_conv_as_batched_gemm_matches_conv1d() -> None: + torch.manual_seed(0) + conv = torch.nn.Conv1d(64, 64, 31, groups=16, padding=15) + x = torch.randn(2, 37, 64) + expected = conv(x.transpose(1, 2)).transpose(1, 2) + torch.testing.assert_close(T._grouped_conv1d_btc(x, conv), expected, atol=1e-5, rtol=1e-5) + + +@torch.inference_mode() +@pytest.mark.parametrize("masked", [False, True]) +def test_conv_pos_embedding_matches_the_channels_first_stack(masked: bool) -> None: + torch.manual_seed(1) + module = T.ConvPosEmbedding(64).eval() + x = torch.randn(2, 37, 64) + mask = None + if masked: + mask = torch.ones(2, 37, dtype=torch.bool) + mask[1, 30:] = False + torch.testing.assert_close(module(x, mask), _reference_conv_pos_embed(module, x, mask), atol=1e-5, rtol=1e-5) + + +@torch.inference_mode() +@pytest.mark.parametrize("cfg_infer", [False, True]) +def test_modulation_table_matches_per_step_modulation(cfg_infer: bool) -> None: + dit = _make_dit() + text = torch.randn(1, 5, 8) + ref = torch.randn(1, 3, 4) + x = torch.randn(1, 6, 4) + grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cpu") + plain = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer) + tabled = dit.prepare(text, target_len=6, ref=ref, cfg_infer=cfg_infer, timesteps=grid[:-1]) + assert tabled.modulation is not None and tabled.modulation.shape[0] == 4 + for i in range(4): + expected = dit.step(x, grid[i], plain) + torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=i), expected, atol=1e-5, rtol=1e-5) + index = torch.tensor(i) + torch.testing.assert_close(dit.step(x, grid[i], tabled, step_index=index), expected, atol=1e-5, rtol=1e-5) + + +def _padded_inputs(device: str) -> dict[str, torch.Tensor]: + generator = torch.Generator(device="cpu").manual_seed(3) + text = torch.randn(1, 7, 8, generator=generator) + c_mask = torch.ones(1, 7, dtype=torch.bool) + c_mask[:, 5:] = False + text[:, 5:] = 0.0 + ref = torch.randn(1, 6, 4, generator=generator) + ref_mask = torch.ones(1, 6, dtype=torch.bool) + ref_mask[:, 4:] = False + mask = torch.ones(1, 9, dtype=torch.bool) + mask[:, 7:] = False + inputs = {"text": text, "c_mask": c_mask, "ref": ref, "ref_mask": ref_mask, "mask": mask} + return {name: value.to(device) for name, value in inputs.items()} + + +@pytest.mark.cuda +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA") +@torch.inference_mode() +@pytest.mark.parametrize("cfg_strength", [0.0, 2.0]) +def test_graph_with_time_grid_matches_eager_with_time_grid(cfg_strength: float) -> None: + dit = _make_dit("cuda") + inputs = _padded_inputs("cuda") + grid = build_time_grid(nfe=4, sway_sampling_coef=-1.0, t_grid=None, device="cuda") + x = torch.randn(1, 9, 4, device="cuda") + wrapper = AuKCUDAGraphWrapper(dit) + common = dict( + text=inputs["text"], + c_mask=inputs["c_mask"], + ref=inputs["ref"], + ref_mask=inputs["ref_mask"], + cfg_strength=cfg_strength, + timesteps=grid[:-1], + ) + for i in range(4): + graph = wrapper(x=x, timestep=grid[i], new_request=i == 0, step_index=i, **common) + ctx = dit.prepare( + inputs["text"], + target_len=9, + c_mask=inputs["c_mask"], + ref=inputs["ref"], + ref_mask=inputs["ref_mask"], + cfg_infer=cfg_strength >= 1e-5, + timesteps=grid[:-1], + ) + eager = wrapper._step(x, grid[i], ctx, cfg_strength, i) + torch.testing.assert_close(graph, eager, atol=1e-4, rtol=1e-4) + # One graph serves every step of the grid. + assert len(wrapper._cache) == 1 + assert next(iter(wrapper._cache))[-1] == 4 diff --git a/tests/diffusion/models/auk/test_auk_step_context.py b/tests/diffusion/models/auk/test_auk_step_context.py index e70d52510..26f722fe3 100644 --- a/tests/diffusion/models/auk/test_auk_step_context.py +++ b/tests/diffusion/models/auk/test_auk_step_context.py @@ -127,7 +127,9 @@ def test_hoisted_euler_loop_matches_per_step_forward(cfg_strength: float) -> Non v = dit(x, time=timesteps[i], **inputs) x = x + (timesteps[i + 1] - timesteps[i]) * v - torch.testing.assert_close(got, x, rtol=0, atol=0) + # sample_latents precomputes the adaLN modulations of the whole grid in one + # GEMM per layer, whose rows round differently from one-row GEMMs in fp32. + torch.testing.assert_close(got, x, rtol=0, atol=1e-6) assert dit.text_cond is None and dit.text_uncond is None diff --git a/vllm_omni/diffusion/models/auk/auk_transformer.py b/vllm_omni/diffusion/models/auk/auk_transformer.py index 4056c72eb..fb146b101 100644 --- a/vllm_omni/diffusion/models/auk/auk_transformer.py +++ b/vllm_omni/diffusion/models/auk/auk_transformer.py @@ -153,6 +153,33 @@ class TimeEmbedding(nn.Module): return self.time_mlp(hidden.to(self.time_mlp[0].weight.dtype)) +def _grouped_conv1d_btc(x: torch.Tensor, conv: nn.Conv1d) -> torch.Tensor: + """``conv`` applied to channels-last ``x`` ``[B, T, C]``, returning ``[B, T, C]``. + + A same-padded grouped convolution written as one im2col copy and one + batched GEMM over the groups. cuDNN runs a grouped conv of this shape as + one kernel per group plus two layout transposes, which at AuK's sizes + (1536 channels in 16 groups, kernel 31, a few hundred frames) costs about + ten times the arithmetic. + """ + batch, frames, channels = x.shape + groups = conv.groups + kernel = conv.kernel_size[0] + per_group = channels // groups + pad = conv.padding[0] + padded = F.pad(x, (0, 0, pad, pad)) + # unfold gives a [B, T, C, K] view; one copy lays it out as [B, T, G, C/G * K]. + cols = padded.unfold(1, kernel, 1).reshape(batch * frames, groups, per_group * kernel) + # conv.weight [C_out, C/G, K] -> [G, C/G * K, C_out/G]; the reduction order matches the columns. + weight = conv.weight.reshape(groups, channels // groups, per_group * kernel).transpose(1, 2).to(cols.dtype) + # [G, B*T, C/G*K] @ [G, C/G*K, C_out/G] -> [G, B*T, C_out/G] + out = torch.bmm(cols.transpose(0, 1), weight) + out = out.transpose(0, 1).reshape(batch, frames, channels) + if conv.bias is not None: + out = out + conv.bias.to(out.dtype) + return out + + class ConvPosEmbedding(nn.Module): """Depthwise-grouped conv stack that adds local position information.""" @@ -169,17 +196,19 @@ class ConvPosEmbedding(nn.Module): ) def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor: - keep = None if mask is None else mask.unsqueeze(1) - x = x.transpose(1, 2) + keep = None if mask is None else mask.unsqueeze(-1) if keep is not None: x = x.masked_fill(~keep, 0.0) for layer in self.conv1d: - x = layer(x) - # Re-zero padding after each convolution so it cannot bleed into - # valid frames through the kernel window. - if keep is not None and isinstance(layer, nn.Conv1d): - x = x.masked_fill(~keep, 0.0) - return x.transpose(1, 2) + if isinstance(layer, nn.Conv1d): + x = _grouped_conv1d_btc(x, layer) + # Re-zero padding after each convolution so it cannot bleed into + # valid frames through the kernel window. + if keep is not None: + x = x.masked_fill(~keep, 0.0) + else: + x = layer(x) + return x class AudioEmbedding(nn.Module): @@ -205,9 +234,12 @@ class AdaLayerNorm(nn.Module): self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) def forward( - self, x: torch.Tensor, emb: torch.Tensor + self, x: torch.Tensor, emb: torch.Tensor | None, mod: torch.Tensor | None = None ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.linear(self.silu(emb)).chunk(6, dim=1) + """``mod`` is the precomputed ``linear(silu(emb))``; ``emb`` is then unused.""" + if mod is None: + mod = self.linear(self.silu(emb)) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=1) x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] return x, gate_msa, shift_mlp, scale_mlp, gate_mlp @@ -221,8 +253,11 @@ class AdaLayerNormFinal(nn.Module): self.linear = nn.Linear(dim, dim * 2) self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) - def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor: - scale, shift = self.linear(self.silu(emb)).chunk(2, dim=1) + def forward(self, x: torch.Tensor, emb: torch.Tensor | None, mod: torch.Tensor | None = None) -> torch.Tensor: + """``mod`` is the precomputed ``linear(silu(emb))``; ``emb`` is then unused.""" + if mod is None: + mod = self.linear(self.silu(emb)) + scale, shift = mod.chunk(2, dim=1) return self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :] @@ -383,9 +418,11 @@ class DoubleBlock(nn.Module): c_rope: RopeCache, c_mask: torch.Tensor | None, bias: torch.Tensor | None = None, + mod_c: torch.Tensor | None = None, + mod_x: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: - norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t) - norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t) + norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, t, mod_c) + norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, t, mod_x) x_attn, c_attn = self.attn(x=norm_x, c=norm_c, mask=mask, rope=rope, c_rope=c_rope, c_mask=c_mask, bias=bias) @@ -416,8 +453,9 @@ class SingleBlock(nn.Module): mask: torch.Tensor | None, rope: RopeCache, bias: torch.Tensor | None = None, + mod: torch.Tensor | None = None, ) -> torch.Tensor: - norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t) + norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, t, mod) x = x + gate_msa[:, None] * self.attn(x=norm, mask=mask, rope=rope, bias=bias) norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None] return x + gate_mlp[:, None] * self.ff(norm) @@ -491,6 +529,28 @@ class AuKTransformer(nn.Module): self.norm_out = AdaLayerNormFinal(dim) self.proj_out = nn.Linear(dim, latent_dim) + # Widths of the adaLN modulations in the order modulation_table() lays them out. + self._modulation_widths = [module.linear.out_features for module in self._adaln_modules()] + + def _adaln_modules(self) -> list[nn.Module]: + """Every timestep-conditioned norm, in the order :meth:`step` consumes them.""" + modules: list[nn.Module] = [] + for block in self.transformer_blocks: + modules += [block.attn_norm_c, block.attn_norm_x] + modules += [block.attn_norm for block in self.single_transformer_blocks] + modules.append(self.norm_out) + return modules + + def modulation_table(self, timesteps: torch.Tensor) -> torch.Tensor: + """adaLN modulations for every timestep, ``[len(timesteps), sum of widths]``. + + The modulations depend only on the timestep, and a request's time grid + is known before its first step. Computing them for the whole grid at + once reads each modulation weight once per request instead of once per + step, which at batch 1 is what those matrix-vector products cost. + """ + emb = F.silu(self.time_embed(timesteps.reshape(-1).float())) + return torch.cat([module.linear(emb) for module in self._adaln_modules()], dim=1) def project_text(self, text: torch.Tensor) -> torch.Tensor: """Project LLM hidden states ``[B, nt, text_hidden_dim]`` to model width.""" @@ -520,6 +580,7 @@ class AuKTransformer(nn.Module): drop_text: bool = False, cfg_infer: bool = False, cache: bool = False, + timesteps: torch.Tensor | None = None, ) -> "AuKStepContext": """Everything a denoise step needs that does not depend on ``x`` or the time. @@ -528,6 +589,9 @@ class AuKTransformer(nn.Module): prompt embedding, the padding masks and biases and the rotary tables are not recomputed on every step. Arguments are those of :meth:`forward`; ``target_len`` is the target frame count ``n``. + ``timesteps`` are the times the steps will run at (the grid without + its end point); when given, their adaLN modulations are precomputed + and :meth:`step` selects them by ``step_index``. """ batch = text.shape[0] if c_mask is None: @@ -593,25 +657,44 @@ class AuKTransformer(nn.Module): rope_text=_rope_cos_sin(self.rotary_embed(text_len, c_mask)), rope_single=_rope_cos_sin(self.rotary_embed(text_len + audio_len, single_mask)), branches=branches, + modulation=None if timesteps is None else self.modulation_table(timesteps), ) - def step(self, x: torch.Tensor, time: torch.Tensor, ctx: "AuKStepContext") -> torch.Tensor: + def step( + self, + x: torch.Tensor, + time: torch.Tensor, + ctx: "AuKStepContext", + step_index: torch.Tensor | int | None = None, + ) -> torch.Tensor: """Predict the velocity of ``x`` at ``time`` under a :meth:`prepare` context. The target embedding does not depend on the CFG branch, so it is - computed once and shared by both branches. + computed once and shared by both branches. When the context carries a + modulation table, ``step_index`` selects this step's row of it and + ``time`` is unused; the row broadcasts over the CFG branches. """ - if time.ndim == 0: - time = time.repeat(x.shape[0]) - t = self.time_embed(time) target = self.audio_embed(x, mask=ctx.target_mask) + if ctx.modulation is not None and step_index is not None: + if isinstance(step_index, int): + row = ctx.modulation[step_index : step_index + 1] + else: + row = ctx.modulation.index_select(0, step_index.reshape(1)) + mods = list(torch.split(row, self._modulation_widths, dim=1)) + t = None + else: + if time.ndim == 0: + time = time.repeat(x.shape[0]) + t = self.time_embed(time) + if ctx.branches == 2: + t = torch.cat((t, t), dim=0) + mods = [None] * len(self._modulation_widths) if ctx.branches == 2: - t = torch.cat((t, t), dim=0) target = torch.cat((target, target), dim=0) audio = target if ctx.prompt is None else torch.cat([ctx.prompt, target], dim=1) c = ctx.c - for block in self.transformer_blocks: + for i, block in enumerate(self.transformer_blocks): c, audio = block( audio, c, @@ -621,14 +704,24 @@ class AuKTransformer(nn.Module): c_rope=ctx.rope_text, c_mask=ctx.c_mask, bias=ctx.joint_bias, + mod_c=mods[2 * i], + mod_x=mods[2 * i + 1], ) h = torch.cat([c, audio], dim=1) - for block in self.single_transformer_blocks: - h = block(h, t, mask=ctx.single_mask, rope=ctx.rope_single, bias=ctx.single_bias) + offset = 2 * len(self.transformer_blocks) + for i, block in enumerate(self.single_transformer_blocks): + h = block( + h, + t, + mask=ctx.single_mask, + rope=ctx.rope_single, + bias=ctx.single_bias, + mod=mods[offset + i], + ) h = h[:, h.shape[1] - x.shape[1] :] - return self.proj_out(self.norm_out(h, t)) + return self.proj_out(self.norm_out(h, t, mods[-1])) def forward( self, @@ -706,6 +799,8 @@ class AuKStepContext: rope_text: RopeCache rope_single: RopeCache branches: int + #: adaLN modulations per step, ``[steps, widths]``, when the time grid was known. + modulation: torch.Tensor | None = None def tensors(self) -> list[torch.Tensor | None]: """Every tensor field in a fixed order, rope pairs flattened.""" @@ -721,6 +816,7 @@ class AuKStepContext: *self.rope_audio, *self.rope_text, *self.rope_single, + self.modulation, ] def copy_(self, other: "AuKStepContext") -> None: @@ -752,6 +848,7 @@ class AuKStepContext: rope_text=(self.rope_text[0].clone(), self.rope_text[1].clone()), rope_single=(self.rope_single[0].clone(), self.rope_single[1].clone()), branches=self.branches, + modulation=_clone(self.modulation), ) @@ -881,6 +978,8 @@ def sample_latents( timestep=t[i], cfg_strength=cfg_strength, new_request=i == 0, + timesteps=t[:-1], + step_index=i, ) x = x + (t[i + 1] - t[i]) * velocity finally: @@ -922,9 +1021,10 @@ def _sample_latents( ref_mask=ref_mask, cfg_infer=guided, cache=guided, + timesteps=timesteps[:-1], ) for i in range(timesteps.shape[0] - 1): - v = dit.step(x, timesteps[i], ctx) + v = dit.step(x, timesteps[i], ctx, step_index=i) if guided: v_cond, v_uncond = v.chunk(2, dim=0) v = v_cond + (v_cond - v_uncond) * cfg_strength diff --git a/vllm_omni/diffusion/models/auk/cudagraph_wrapper.py b/vllm_omni/diffusion/models/auk/cudagraph_wrapper.py index 7d34557f6..8bf09c2e4 100644 --- a/vllm_omni/diffusion/models/auk/cudagraph_wrapper.py +++ b/vllm_omni/diffusion/models/auk/cudagraph_wrapper.py @@ -24,16 +24,19 @@ class _GraphEntry: static_ctx: AuKStepContext static_timestep: torch.Tensor static_cfg: torch.Tensor | None + static_step: torch.Tensor | None static_out: torch.Tensor class AuKCUDAGraphWrapper: """Replay one DiT denoise step and leave Euler scheduling to the caller. - The graph is keyed by the target, text and reference sequence lengths plus - whether the CFG branch is enabled. Timestep and the CFG strength are + The graph is keyed by the target, text and reference sequence lengths, + whether the CFG branch is enabled and, when the caller passes the time + grid, its length. Timestep (or the step index), and the CFG strength are mutable scalar buffers, so all Euler steps and CFG values within one path - reuse one graph. + reuse one graph. With the time grid, the adaLN modulations of every step + are part of the per-request context and the graph selects its row. Only :meth:`AuKTransformer.step` is captured. The per-request context (text projection, reference embedding, padding biases, rotary tables) is @@ -85,8 +88,9 @@ class AuKCUDAGraphWrapper: text: torch.Tensor, ref: torch.Tensor, uses_cfg: bool, - ) -> tuple[int, int, int, bool]: - return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg) + steps: int = 0, + ) -> tuple[int, int, int, bool, int]: + return (x.shape[1], text.shape[1], ref.shape[1], uses_cfg, steps) def _retire_graph_generation_if_full(self) -> None: """Retire all graphs together so none outlive shared workspaces.""" @@ -103,6 +107,7 @@ class AuKCUDAGraphWrapper: ref: torch.Tensor, ref_mask: torch.Tensor, uses_cfg: bool, + timesteps: torch.Tensor | None = None, ) -> AuKStepContext: return self.dit.prepare( text, @@ -112,6 +117,7 @@ class AuKCUDAGraphWrapper: ref=ref, ref_mask=ref_mask, cfg_infer=uses_cfg, + timesteps=timesteps, ) def _step( @@ -120,9 +126,10 @@ class AuKCUDAGraphWrapper: timestep: torch.Tensor, ctx: AuKStepContext, cfg_strength: torch.Tensor | float, + step_index: torch.Tensor | int | None = None, ) -> torch.Tensor: """One guided (or unguided) velocity; the CFG combine is part of the graph.""" - velocity = self.dit.step(x, timestep, ctx) + velocity = self.dit.step(x, timestep, ctx, step_index=step_index) if ctx.branches == 1: return velocity conditional, unconditional = velocity.chunk(2, dim=0) @@ -140,6 +147,8 @@ class AuKCUDAGraphWrapper: timestep: torch.Tensor, cfg_strength: float, new_request: bool = True, + timesteps: torch.Tensor | None = None, + step_index: int | None = None, ) -> torch.Tensor: """Velocity of ``x`` at ``timestep``. @@ -147,26 +156,33 @@ class AuKCUDAGraphWrapper: (``text``, ``c_mask``, ``ref``, ``ref_mask``) is then prepared anew; the later steps of the request pass ``False`` and reuse it. The default prepares on every call, which is always correct. + + ``timesteps`` (the times every step of the request runs at) and + ``step_index`` (this step's position in them) let the adaLN + modulations be computed once per request instead of once per step. """ uses_cfg = cfg_strength >= 1e-5 + if timesteps is None or step_index is None: + timesteps, step_index = None, None if not self.enabled or x.device.type != "cuda" or torch.cuda.is_current_stream_capturing(): if new_request or self._eager_ctx is None: - self._eager_ctx = self._prepare(x, None, text, c_mask, ref, ref_mask, uses_cfg) - return self._step(x, timestep, self._eager_ctx, cfg_strength) + self._eager_ctx = self._prepare(x, None, text, c_mask, ref, ref_mask, uses_cfg, timesteps) + return self._step(x, timestep, self._eager_ctx, cfg_strength, step_index) target_frames = x.shape[1] x, x_mask, text, c_mask, ref, ref_mask = self._bucket_inputs(x, text, c_mask, ref, ref_mask) - key = self._key(x, text, ref, uses_cfg) + steps = 0 if timesteps is None else int(timesteps.numel()) + key = self._key(x, text, ref, uses_cfg, steps) entry = self._cache.get(key) ctx = None if entry is None or new_request or key != self._active_key: - ctx = self._prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg) + ctx = self._prepare(x, x_mask, text, c_mask, ref, ref_mask, uses_cfg, timesteps) if entry is None: # Same Retirement as #6587, details discussed in #7469 self._retire_graph_generation_if_full() try: assert ctx is not None - entry = self._capture(x, ctx, timestep, cfg_strength=cfg_strength) + entry = self._capture(x, ctx, timestep, cfg_strength=cfg_strength, step_index=step_index) except Exception: self._cache.clear() self._active_key = None @@ -186,6 +202,8 @@ class AuKCUDAGraphWrapper: entry.static_x.copy_(x) entry.static_timestep.copy_(timestep) + if entry.static_step is not None: + entry.static_step.fill_(step_index) if entry.static_cfg is not None: entry.static_cfg.fill_(cfg_strength) entry.graph.replay() @@ -198,10 +216,14 @@ class AuKCUDAGraphWrapper: timestep: torch.Tensor, *, cfg_strength: float, + step_index: int | None = None, ) -> _GraphEntry: static_x = x.clone() static_ctx = ctx.clone() static_timestep = timestep.clone() + static_step = None + if static_ctx.modulation is not None and step_index is not None: + static_step = torch.full((), step_index, device=static_x.device, dtype=torch.long) static_cfg = None if ctx.branches == 2: static_cfg = torch.empty((), device=static_x.device, dtype=torch.float32) @@ -209,13 +231,13 @@ class AuKCUDAGraphWrapper: scale = static_cfg if static_cfg is not None else cfg_strength # Warm-up runs any lazy torch.compile outside the capture. for _ in range(3): - self._step(static_x, static_timestep, static_ctx, scale) + self._step(static_x, static_timestep, static_ctx, scale, static_step) if self._pool_handle is None: self._pool_handle = torch.cuda.graph_pool_handle() graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, pool=self._pool_handle): # The CFG scalar stays a mutable buffer, so cfg=2 and cfg=3 share this graph. - static_out = self._step(static_x, static_timestep, static_ctx, scale) + static_out = self._step(static_x, static_timestep, static_ctx, scale, static_step) logger.info( "Captured AuK DiT single-step CUDA graph: target_frames=%d text_tokens=%d ref_frames=%d cfg=%s", @@ -230,6 +252,7 @@ class AuKCUDAGraphWrapper: static_ctx=static_ctx, static_timestep=static_timestep, static_cfg=static_cfg, + static_step=static_step, static_out=static_out, ) diff --git a/vllm_omni/diffusion/models/auk/pipeline_auk.py b/vllm_omni/diffusion/models/auk/pipeline_auk.py index 5e74c491a..da5d8d1e5 100644 --- a/vllm_omni/diffusion/models/auk/pipeline_auk.py +++ b/vllm_omni/diffusion/models/auk/pipeline_auk.py @@ -27,7 +27,12 @@ from vllm.logger import init_logger from vllm_omni.diffusion.compile import regionally_compile from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.utils import get_local_device -from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, dit_state_dict, sample_latents +from vllm_omni.diffusion.models.auk.auk_transformer import ( + AuKTransformer, + build_time_grid, + dit_state_dict, + sample_latents, +) from vllm_omni.diffusion.models.auk.auk_vae import AuKVAE from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper from vllm_omni.diffusion.models.auk.vae_cudagraph import AuKVAEDecodeGraph @@ -267,6 +272,12 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp text = torch.zeros(1, _WARMUP_TEXT_TOKENS, self.text_hidden_dim, device=self.device, dtype=self.dtype) c_mask = torch.ones(text.shape[:2], dtype=torch.bool, device=self.device) timestep = torch.zeros((), device=self.device, dtype=torch.float32) + # The graphs are keyed by the number of steps, so warm the variant's default grid. + if self.is_flash: + grid = torch.tensor(self.flash_t_grid, device=self.device, dtype=torch.float32) + else: + sway = None if self.default_sway is None else float(self.default_sway) + grid = build_time_grid(nfe=self.default_nfe, sway_sampling_coef=sway, t_grid=None, device=self.device) start = time.perf_counter() with torch.inference_mode(), self._dit_autocast(): for ref_frames in _WARMUP_REF_FRAMES: @@ -283,6 +294,8 @@ class AuKPipeline(nn.Module, SupportAudioInput, SupportAudioOutput, SupportsComp timestep=timestep, cfg_strength=cfg, new_request=True, + timesteps=grid[:-1], + step_index=0, ) if self.device.type != "cpu": torch.accelerator.synchronize(self.device)