Optimize Qwen-Image 2.1 attention and decode graph reuse (#7790)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
Tianyu Guo authored and GitHub committed 2026-09-18 19:37:19 +08:00
1 parent f930348e79
commit ab4cf24b3e
24 files changed
+1147 -414

No files matched your search

@@ -9,19 +9,14 @@ default cache stores prefix K/V in freshly allocated tensors, so a captured
graph would bind addresses that the next request (or a step-mode cache merge)
no longer owns.
This module keeps the prefix K/V in persistent, fixed-address buffers. Prefill
copies its K/V into them and re-points the caller's cache dict at the buffers;
decode then replays a captured graph that reads those buffers plus static input
buffers (target latents, timestep, RoPE frequencies). One graph is captured per
``(branch, batch, prefix_len, target_tokens, dtype, device, backend)`` key and
reused across requests: a new prefill with the same key rewrites the buffers in
place, which a captured graph reads correctly because it binds addresses, not
contents.
Requests retain ownership of their prefix K/V. Decode refreshes separate
fixed-address graph buffers when the active prefix tensors change. Graphs
are keyed by branch, batch, prefix length, complete image layout,
dtype, device and backend, so equal token counts with different RoPE layouts stay separate.
Memory note: only the *prefix* K/V are persistent. The decode attention still
concatenates prefix K/V with the current step's target K/V inside the captured
region, and that transient tensor is served by the graph's memory pool, so the
persistent footprint is the same as the existing dict cache.
Memory note: graph entries own an additional copy of the prefix K/V. Decode
attention concatenates that prefix with target K/V inside the captured region;
transient tensors use the graph memory pool.
Fallbacks (all logged, all eager): CUDA unavailable, sequence/ring/tensor
parallelism, HSDP/offload hooks, torch.compile'd blocks, KV-cache quantization,
@@ -32,6 +27,7 @@ capture failure.
from __future__ import annotations
import weakref
from collections import OrderedDict
from typing import TYPE_CHECKING
@@ -39,6 +35,8 @@ import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm_omni.platforms import current_omni_platform
if TYPE_CHECKING:
from vllm_omni.diffusion.models.qwen_image_21.qwen_image_21_transformer import (
QwenImage21Transformer2DModel,
@@ -67,8 +65,8 @@ class QwenImage21DecodeGraphEntry:
Buffers are created outside inference mode so replay-side ``copy_`` is
legal regardless of the caller's grad context. ``graph`` is None until the
first decode step captures it; ``failed`` disables capture permanently for
this key after an exception.
first decode step captures it. Failed entries are replaced by a ``None``
marker in the manager so their buffers can be released.
"""
def __init__(
@@ -99,15 +97,30 @@ class QwenImage21DecodeGraphEntry:
# inference_mode, and the captured body takes views of this tensor
# outside it, which is forbidden for inference tensors.
self.freqs = freqs.clone()
# Decode needs no mask once padding is excluded (see the manager's
# prefill gate), so the captured body attends without metadata.
self.attn_metadata = None
self.attn_metadata = None
# Per-block cache dicts with the same shape as the legacy protocol.
self.block_caches = [{branch: {"key": self.k[i], "value": self.v[i]}} for i in range(len(kv_shapes))]
self.graph: torch.cuda.CUDAGraph | None = None
self.output: torch.Tensor | None = None
self.failed = False
self.captures = 0
self.prefix_sources: list[tuple[weakref.ReferenceType[torch.Tensor], int | None]] = []
def refresh_prefix(self, kv_cache: list[dict[str, dict[str, torch.Tensor]]]) -> None:
sources = [block[self.branch][name] for block in kv_cache for name in ("key", "value")]
versions = [None if tensor.is_inference() else tensor._version for tensor in sources]
if len(sources) == len(self.prefix_sources) and all(
previous() is tensor and version == current_version
for (previous, version), tensor, current_version in zip(self.prefix_sources, sources, versions)
):
return
# Prefill replaces inference-mode tensors; decode treats them as immutable.
# Weak references avoid retaining completed requests or mistaking a reused
# allocation for the same prefix. Normal tensors also track in-place writes.
self.prefix_sources = []
for i, block in enumerate(kv_cache):
self.k[i].copy_(block[self.branch]["key"])
self.v[i].copy_(block[self.branch]["value"])
self.prefix_sources = [(weakref.ref(tensor), version) for tensor, version in zip(sources, versions)]
def capture(self, body) -> None:
"""Warm up on a side stream, then capture one decode step.
@@ -119,10 +132,12 @@ class QwenImage21DecodeGraphEntry:
"""
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
for _ in range(2):
body()
torch.cuda.current_stream().wait_stream(side)
try:
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
for _ in range(2):
body()
finally:
torch.cuda.current_stream().wait_stream(side)
graph = torch.cuda.CUDAGraph()
# The platform-wide pool is shared with every other captured path in
@@ -142,7 +157,7 @@ class QwenImage21DecodeGraphManager:
def __init__(self, model: QwenImage21Transformer2DModel, max_entries: int = 8):
self.model = model
self.max_entries = max_entries
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry] = OrderedDict()
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry | None] = OrderedDict()
self._static_eligible: bool | None = None
# ── eligibility ──
@@ -202,17 +217,12 @@ class QwenImage21DecodeGraphManager:
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
cache_branch: str,
prefix_len: int,
target_tokens: int,
img_shapes: list[tuple[int, int, int]],
target_freqs: torch.Tensor,
joint_key_valid: torch.Tensor | None,
dtype: torch.dtype,
) -> None:
"""Move a freshly prefilled prefix K/V into static buffers.
The caller's ``kv_cache`` dicts are re-pointed at the static buffers, so
the existing decode path reads fixed addresses and a captured graph can
be replayed for later steps. Prefill itself always runs eagerly.
"""
"""Prepare graph buffers without changing the request's prefix K/V."""
if not self.eligible() or torch.compiler.is_compiling():
return
if _dynamic_lora_wrappers_present(self.model):
@@ -239,44 +249,55 @@ class QwenImage21DecodeGraphManager:
cache_branch,
batch_size,
prefix_len,
target_tokens,
tuple(tuple(shape) for shape in img_shapes),
dtype,
first["key"].device.index,
self._backend_name(),
)
entry = self.entries.get(key)
if entry is None:
if key not in self.entries:
while len(self.entries) >= self.max_entries:
evicted_key, _ = self.entries.popitem(last=False)
evicted_key, evicted_entry = self.entries.popitem(last=False)
del evicted_entry
logger.debug("Evicting decode graph entry %s (max_entries=%d)", evicted_key, self.max_entries)
entry = QwenImage21DecodeGraphEntry(
branch=cache_branch,
batch_size=batch_size,
prefix_len=prefix_len,
target_tokens=target_tokens,
in_channels=self.model.in_channels,
dtype=dtype,
device=first["key"].device,
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
kv_dtype=first["key"].dtype,
freqs=target_freqs,
)
self.entries[key] = entry
allocation_error = None
try:
self.entries[key] = QwenImage21DecodeGraphEntry(
branch=cache_branch,
batch_size=batch_size,
prefix_len=prefix_len,
target_tokens=target_freqs.shape[0],
in_channels=self.model.in_channels,
dtype=dtype,
device=first["key"].device,
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
kv_dtype=first["key"].dtype,
freqs=target_freqs,
)
except torch.OutOfMemoryError as exc:
allocation_error = str(exc)
if allocation_error is not None:
# Keep a bounded failure marker without retaining tensors or the exception traceback.
self.entries[key] = None
current_omni_platform.empty_cache()
logger.warning(
"Qwen-Image-2.1 CUDA graph allocation failed for key=%s: %s. Falling back to eager decode.",
key,
allocation_error,
)
return
else:
self.entries.move_to_end(key)
for i, block_cache in enumerate(kv_cache):
parts = block_cache[cache_branch]
entry.k[i].copy_(parts["key"])
entry.v[i].copy_(parts["value"])
parts["key"] = entry.k[i]
parts["value"] = entry.v[i]
entry = self.entries[key]
if entry is not None:
entry.prefix_sources = []
logger.debug(
"Registered decode graph entry key=%s (entries=%d, captured=%d)",
key,
len(self.entries),
sum(1 for e in self.entries.values() if e.graph is not None),
sum(1 for e in self.entries.values() if e is not None and e.graph is not None),
)
# ── decode replay ──
@@ -288,11 +309,20 @@ class QwenImage21DecodeGraphManager:
timestep: torch.Tensor,
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
cache_branch: str,
target_tokens: int,
img_shapes: list[tuple[int, int, int]],
img_mask: torch.Tensor,
encoder_hidden_states_mask: torch.Tensor | None,
) -> torch.Tensor | None:
"""Replay the captured decode step, or return None to fall back to eager."""
if not self.eligible() or torch.compiler.is_compiling():
return None
if _dynamic_lora_wrappers_present(self.model):
self.clear()
return None
if encoder_hidden_states_mask is not None:
text_positions = ~img_mask[0, : encoder_hidden_states_mask.shape[1]]
if not bool(encoder_hidden_states_mask[:, text_positions].all()):
return None
first = kv_cache[0][cache_branch]
cached_key = first["key"]
if cached_key.device.type != "cuda":
@@ -304,40 +334,36 @@ class QwenImage21DecodeGraphManager:
cache_branch,
cached_key.shape[0],
cached_key.shape[1],
target_tokens,
tuple(tuple(shape) for shape in img_shapes),
hidden_states.dtype,
cached_key.device.index,
self._backend_name(),
)
entry = self.entries.get(key)
if entry is None or entry.failed:
if entry is None:
return None
self.entries.move_to_end(key)
entry.hidden.copy_(hidden_states[:, -target_tokens:])
entry.hidden.copy_(hidden_states[:, -entry.target_tokens :])
entry.timestep.copy_(timestep)
# The caller's cache dicts normally already reference the entry's
# buffers (prefill re-pointed them). A step-mode batched decode instead
# hands over freshly merged tensors; copy those into the static buffers.
for i, block_cache in enumerate(kv_cache):
parts = block_cache[cache_branch]
if parts["key"] is not entry.k[i]:
entry.k[i].copy_(parts["key"])
entry.v[i].copy_(parts["value"])
parts["key"] = entry.k[i]
parts["value"] = entry.v[i]
# Graph buffers are scratch space; request caches must never alias them.
entry.refresh_prefix(kv_cache)
if entry.graph is None:
capture_error = None
try:
entry.capture(lambda: self.model._decode_graph_forward(entry))
except Exception as exc:
entry.failed = True
entry.graph = None
capture_error = str(exc)
if capture_error is not None:
self.entries[key] = None
del entry
current_omni_platform.empty_cache()
logger.warning(
"Qwen-Image-2.1 CUDA graph capture failed for key=%s: %s. "
"Falling back to eager decode for this shape.",
key,
exc,
capture_error,
)
return None
logger.info("Captured Qwen-Image-2.1 decode graph for key=%s", key)
@@ -144,7 +144,7 @@ def get_qwen_image_21_pre_process_func(
if "additional_information" not in prompt:
prompt["additional_information"] = {}
if not raw_image: # None or empty list: pure text-to-image request
if raw_image is None or (isinstance(raw_image, list) and len(raw_image) == 0):
request.batch_compatibility_key = ("qwen_image_21", ())
request.prompt = prompt
return request
@@ -744,6 +744,7 @@ class QwenImage21Pipeline(
timesteps, num_inference_steps = retrieve_timesteps(
self.scheduler,
num_inference_steps,
device=self.device,
sigmas=sigmas,
mu=mu,
)
@@ -13,6 +13,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from diffusers.models.modeling_outputs import Transformer2DModelOutput
from diffusers.models.normalization import RMSNorm
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -107,7 +108,10 @@ class QwenImage21TemporalTimesteps(nn.Module):
self.time_factor = time_factor
half = timestep_dim // 2
freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half)
# Compute the fixed table on CPU like Diffusers, then honor the loader's device.
freqs = torch.exp(
-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device="cpu") / half
).to(torch.get_default_device())
self.register_buffer("freqs", freqs, persistent=False)
def forward(self, timestep: torch.Tensor) -> torch.Tensor:
@@ -294,12 +298,15 @@ class QwenImage21Rope(nn.Module):
self.theta = theta
self.axes_dim = axes_dim
pos_index = torch.arange(8192)
neg_index = torch.arange(1024).flip(0) * -1 - 1
self.freqs = [
torch.cat([self.rope_params(pos_index, dim, theta), self.rope_params(neg_index, dim, theta)], dim=0)
for dim in axes_dim
]
# Match Diffusers' CPU initialization even when the loader sets a CUDA
# default device; CPU and CUDA trigonometric kernels round differently.
with torch.device("cpu"):
pos_index = torch.arange(8192)
neg_index = torch.arange(1024).flip(0) * -1 - 1
self.freqs = [
torch.cat([self.rope_params(pos_index, dim, theta), self.rope_params(neg_index, dim, theta)], dim=0)
for dim in axes_dim
]
def rope_params(self, index: torch.Tensor, dim: int, theta: int = 10000) -> torch.Tensor:
freqs = torch.outer(index, 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)))
@@ -374,9 +381,10 @@ class QwenImage21Attention(nn.Module):
self.num_heads = self.to_qkv.num_heads
self.num_kv_heads = self.to_qkv.num_kv_heads
# Per-head RMSNorm over the head dim; TP shards whole heads, so the norm is TP-safe.
self.norm_q = nn.RMSNorm(dim_head, eps=eps)
self.norm_k = nn.RMSNorm(dim_head, eps=eps)
# Diffusers normalizes in FP32, then casts before the learned scale.
# Keep that rounding order for BF16 Q/K; TP shards whole heads.
self.norm_q = RMSNorm(dim_head, eps=eps)
self.norm_k = RMSNorm(dim_head, eps=eps)
self.to_out = RowParallelLinear(
heads * dim_head,
@@ -718,7 +726,6 @@ class QwenImage21Transformer2DModel(CachedTransformer):
causal_block: bool = True,
quant_config: QuantizationConfig | None = None,
prefix: str = "transformer",
enable_cuda_graph_decode: bool | None = None,
cuda_graph_max_decode_graphs: int = 8,
):
super().__init__()
@@ -790,14 +797,11 @@ class QwenImage21Transformer2DModel(CachedTransformer):
# Module boundary where _sp_plan shards the target tokens + their RoPE freqs.
self.sequence_prepare = QwenImage21SequencePrepare(self.img_in, self.pos_embed)
# Opt-in CUDA Graph capture of the fixed-shape KV-cache decode steps.
# Default off: ``None`` defers to ``od_config.enable_cuda_graph_decode``.
if enable_cuda_graph_decode is None:
enable_cuda_graph_decode = bool(getattr(od_config, "enable_cuda_graph_decode", False))
self.enable_cuda_graph_decode = enable_cuda_graph_decode
# Use decode graphs unless the caller requests eager execution.
self.enable_cuda_graph_decode = not od_config.enforce_eager
self._decode_graph_manager = (
QwenImage21DecodeGraphManager(self, max_entries=cuda_graph_max_decode_graphs)
if enable_cuda_graph_decode
if self.enable_cuda_graph_decode
else None
)
@@ -972,7 +976,9 @@ class QwenImage21Transformer2DModel(CachedTransformer):
timestep=timestep,
kv_cache=kv_cache,
cache_branch=cache_branch,
target_tokens=math.prod(layout[-1]),
img_shapes=layout,
img_mask=img_mask,
encoder_hidden_states_mask=encoder_hidden_states_mask,
)
if graph_output is not None:
if not return_dict:
@@ -1039,6 +1045,9 @@ class QwenImage21Transformer2DModel(CachedTransformer):
joint_key_valid = F.pad(joint_key_valid, (0, sp_padding), value=False)
has_padding = joint_key_valid is not None and not bool(joint_key_valid.all())
if not has_padding:
# Match Diffusers and use the same mask-free dispatch in eager and graph decode.
joint_key_valid = None
attn_metadata: AttentionMetadata | None = None
cache_write_len: int | None = None
if is_decode:
@@ -1082,13 +1091,12 @@ class QwenImage21Transformer2DModel(CachedTransformer):
and kv_cache is not None
and self._decode_graph_manager is not None
):
# Move the freshly written prefix K/V into fixed-address buffers so
# decode steps can replay a captured CUDA graph against them.
# Allocate graph scratch buffers; the request keeps its own prefix K/V.
self._decode_graph_manager.register_prefill(
kv_cache=kv_cache,
cache_branch=cache_branch,
prefix_len=cache_write_len,
target_tokens=target_freqs.shape[0],
img_shapes=layout,
target_freqs=target_freqs,
joint_key_valid=joint_key_valid,
dtype=hidden_states.dtype,
@@ -1113,9 +1121,9 @@ class QwenImage21Transformer2DModel(CachedTransformer):
"""The exact decode-step computation, captured into a CUDA graph.
Mirrors the eager decode path of ``forward``: target tokens only,
prefix K/V read from the entry's static buffers, no attention metadata
(the registration gate excludes padded masks), and an all-ones
modulation mask. All inputs live in the entry's fixed-address buffers.
prefix K/V read from the entry's static buffers, the same mask-free
attention dispatch as eager decode, and an all-ones modulation mask.
All inputs live in the entry's fixed-address buffers.
"""
hidden_states = self.img_in(entry.hidden)
timestep = torch.cat([entry.timestep, entry.timestep.new_zeros(1)], dim=0)