Optimize Qwen-Image 2.1 attention and decode graph reuse (#7790)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
1 parent
f930348e79
commit
ab4cf24b3e
24 files changed
+1147
-414
No files matched your search
@@ -9,19 +9,14 @@ default cache stores prefix K/V in freshly allocated tensors, so a captured
|
||||
graph would bind addresses that the next request (or a step-mode cache merge)
|
||||
no longer owns.
|
||||
|
||||
This module keeps the prefix K/V in persistent, fixed-address buffers. Prefill
|
||||
copies its K/V into them and re-points the caller's cache dict at the buffers;
|
||||
decode then replays a captured graph that reads those buffers plus static input
|
||||
buffers (target latents, timestep, RoPE frequencies). One graph is captured per
|
||||
``(branch, batch, prefix_len, target_tokens, dtype, device, backend)`` key and
|
||||
reused across requests: a new prefill with the same key rewrites the buffers in
|
||||
place, which a captured graph reads correctly because it binds addresses, not
|
||||
contents.
|
||||
Requests retain ownership of their prefix K/V. Decode refreshes separate
|
||||
fixed-address graph buffers when the active prefix tensors change. Graphs
|
||||
are keyed by branch, batch, prefix length, complete image layout,
|
||||
dtype, device and backend, so equal token counts with different RoPE layouts stay separate.
|
||||
|
||||
Memory note: only the *prefix* K/V are persistent. The decode attention still
|
||||
concatenates prefix K/V with the current step's target K/V inside the captured
|
||||
region, and that transient tensor is served by the graph's memory pool, so the
|
||||
persistent footprint is the same as the existing dict cache.
|
||||
Memory note: graph entries own an additional copy of the prefix K/V. Decode
|
||||
attention concatenates that prefix with target K/V inside the captured region;
|
||||
transient tensors use the graph memory pool.
|
||||
|
||||
Fallbacks (all logged, all eager): CUDA unavailable, sequence/ring/tensor
|
||||
parallelism, HSDP/offload hooks, torch.compile'd blocks, KV-cache quantization,
|
||||
@@ -32,6 +27,7 @@ capture failure.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -39,6 +35,8 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.models.qwen_image_21.qwen_image_21_transformer import (
|
||||
QwenImage21Transformer2DModel,
|
||||
@@ -67,8 +65,8 @@ class QwenImage21DecodeGraphEntry:
|
||||
|
||||
Buffers are created outside inference mode so replay-side ``copy_`` is
|
||||
legal regardless of the caller's grad context. ``graph`` is None until the
|
||||
first decode step captures it; ``failed`` disables capture permanently for
|
||||
this key after an exception.
|
||||
first decode step captures it. Failed entries are replaced by a ``None``
|
||||
marker in the manager so their buffers can be released.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -99,15 +97,30 @@ class QwenImage21DecodeGraphEntry:
|
||||
# inference_mode, and the captured body takes views of this tensor
|
||||
# outside it, which is forbidden for inference tensors.
|
||||
self.freqs = freqs.clone()
|
||||
# Decode needs no mask once padding is excluded (see the manager's
|
||||
# prefill gate), so the captured body attends without metadata.
|
||||
self.attn_metadata = None
|
||||
self.attn_metadata = None
|
||||
# Per-block cache dicts with the same shape as the legacy protocol.
|
||||
self.block_caches = [{branch: {"key": self.k[i], "value": self.v[i]}} for i in range(len(kv_shapes))]
|
||||
self.graph: torch.cuda.CUDAGraph | None = None
|
||||
self.output: torch.Tensor | None = None
|
||||
self.failed = False
|
||||
self.captures = 0
|
||||
self.prefix_sources: list[tuple[weakref.ReferenceType[torch.Tensor], int | None]] = []
|
||||
|
||||
def refresh_prefix(self, kv_cache: list[dict[str, dict[str, torch.Tensor]]]) -> None:
|
||||
sources = [block[self.branch][name] for block in kv_cache for name in ("key", "value")]
|
||||
versions = [None if tensor.is_inference() else tensor._version for tensor in sources]
|
||||
if len(sources) == len(self.prefix_sources) and all(
|
||||
previous() is tensor and version == current_version
|
||||
for (previous, version), tensor, current_version in zip(self.prefix_sources, sources, versions)
|
||||
):
|
||||
return
|
||||
# Prefill replaces inference-mode tensors; decode treats them as immutable.
|
||||
# Weak references avoid retaining completed requests or mistaking a reused
|
||||
# allocation for the same prefix. Normal tensors also track in-place writes.
|
||||
self.prefix_sources = []
|
||||
for i, block in enumerate(kv_cache):
|
||||
self.k[i].copy_(block[self.branch]["key"])
|
||||
self.v[i].copy_(block[self.branch]["value"])
|
||||
self.prefix_sources = [(weakref.ref(tensor), version) for tensor, version in zip(sources, versions)]
|
||||
|
||||
def capture(self, body) -> None:
|
||||
"""Warm up on a side stream, then capture one decode step.
|
||||
@@ -119,10 +132,12 @@ class QwenImage21DecodeGraphEntry:
|
||||
"""
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
|
||||
for _ in range(2):
|
||||
body()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
try:
|
||||
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
|
||||
for _ in range(2):
|
||||
body()
|
||||
finally:
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
# The platform-wide pool is shared with every other captured path in
|
||||
@@ -142,7 +157,7 @@ class QwenImage21DecodeGraphManager:
|
||||
def __init__(self, model: QwenImage21Transformer2DModel, max_entries: int = 8):
|
||||
self.model = model
|
||||
self.max_entries = max_entries
|
||||
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry] = OrderedDict()
|
||||
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry | None] = OrderedDict()
|
||||
self._static_eligible: bool | None = None
|
||||
|
||||
# ── eligibility ──
|
||||
@@ -202,17 +217,12 @@ class QwenImage21DecodeGraphManager:
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
prefix_len: int,
|
||||
target_tokens: int,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
target_freqs: torch.Tensor,
|
||||
joint_key_valid: torch.Tensor | None,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Move a freshly prefilled prefix K/V into static buffers.
|
||||
|
||||
The caller's ``kv_cache`` dicts are re-pointed at the static buffers, so
|
||||
the existing decode path reads fixed addresses and a captured graph can
|
||||
be replayed for later steps. Prefill itself always runs eagerly.
|
||||
"""
|
||||
"""Prepare graph buffers without changing the request's prefix K/V."""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
@@ -239,44 +249,55 @@ class QwenImage21DecodeGraphManager:
|
||||
cache_branch,
|
||||
batch_size,
|
||||
prefix_len,
|
||||
target_tokens,
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
dtype,
|
||||
first["key"].device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
|
||||
entry = self.entries.get(key)
|
||||
if entry is None:
|
||||
if key not in self.entries:
|
||||
while len(self.entries) >= self.max_entries:
|
||||
evicted_key, _ = self.entries.popitem(last=False)
|
||||
evicted_key, evicted_entry = self.entries.popitem(last=False)
|
||||
del evicted_entry
|
||||
logger.debug("Evicting decode graph entry %s (max_entries=%d)", evicted_key, self.max_entries)
|
||||
entry = QwenImage21DecodeGraphEntry(
|
||||
branch=cache_branch,
|
||||
batch_size=batch_size,
|
||||
prefix_len=prefix_len,
|
||||
target_tokens=target_tokens,
|
||||
in_channels=self.model.in_channels,
|
||||
dtype=dtype,
|
||||
device=first["key"].device,
|
||||
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
|
||||
kv_dtype=first["key"].dtype,
|
||||
freqs=target_freqs,
|
||||
)
|
||||
self.entries[key] = entry
|
||||
allocation_error = None
|
||||
try:
|
||||
self.entries[key] = QwenImage21DecodeGraphEntry(
|
||||
branch=cache_branch,
|
||||
batch_size=batch_size,
|
||||
prefix_len=prefix_len,
|
||||
target_tokens=target_freqs.shape[0],
|
||||
in_channels=self.model.in_channels,
|
||||
dtype=dtype,
|
||||
device=first["key"].device,
|
||||
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
|
||||
kv_dtype=first["key"].dtype,
|
||||
freqs=target_freqs,
|
||||
)
|
||||
except torch.OutOfMemoryError as exc:
|
||||
allocation_error = str(exc)
|
||||
if allocation_error is not None:
|
||||
# Keep a bounded failure marker without retaining tensors or the exception traceback.
|
||||
self.entries[key] = None
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph allocation failed for key=%s: %s. Falling back to eager decode.",
|
||||
key,
|
||||
allocation_error,
|
||||
)
|
||||
return
|
||||
else:
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
for i, block_cache in enumerate(kv_cache):
|
||||
parts = block_cache[cache_branch]
|
||||
entry.k[i].copy_(parts["key"])
|
||||
entry.v[i].copy_(parts["value"])
|
||||
parts["key"] = entry.k[i]
|
||||
parts["value"] = entry.v[i]
|
||||
entry = self.entries[key]
|
||||
if entry is not None:
|
||||
entry.prefix_sources = []
|
||||
|
||||
logger.debug(
|
||||
"Registered decode graph entry key=%s (entries=%d, captured=%d)",
|
||||
key,
|
||||
len(self.entries),
|
||||
sum(1 for e in self.entries.values() if e.graph is not None),
|
||||
sum(1 for e in self.entries.values() if e is not None and e.graph is not None),
|
||||
)
|
||||
|
||||
# ── decode replay ──
|
||||
@@ -288,11 +309,20 @@ class QwenImage21DecodeGraphManager:
|
||||
timestep: torch.Tensor,
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
target_tokens: int,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
img_mask: torch.Tensor,
|
||||
encoder_hidden_states_mask: torch.Tensor | None,
|
||||
) -> torch.Tensor | None:
|
||||
"""Replay the captured decode step, or return None to fall back to eager."""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return None
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
self.clear()
|
||||
return None
|
||||
if encoder_hidden_states_mask is not None:
|
||||
text_positions = ~img_mask[0, : encoder_hidden_states_mask.shape[1]]
|
||||
if not bool(encoder_hidden_states_mask[:, text_positions].all()):
|
||||
return None
|
||||
first = kv_cache[0][cache_branch]
|
||||
cached_key = first["key"]
|
||||
if cached_key.device.type != "cuda":
|
||||
@@ -304,40 +334,36 @@ class QwenImage21DecodeGraphManager:
|
||||
cache_branch,
|
||||
cached_key.shape[0],
|
||||
cached_key.shape[1],
|
||||
target_tokens,
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
hidden_states.dtype,
|
||||
cached_key.device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
entry = self.entries.get(key)
|
||||
if entry is None or entry.failed:
|
||||
if entry is None:
|
||||
return None
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
entry.hidden.copy_(hidden_states[:, -target_tokens:])
|
||||
entry.hidden.copy_(hidden_states[:, -entry.target_tokens :])
|
||||
entry.timestep.copy_(timestep)
|
||||
# The caller's cache dicts normally already reference the entry's
|
||||
# buffers (prefill re-pointed them). A step-mode batched decode instead
|
||||
# hands over freshly merged tensors; copy those into the static buffers.
|
||||
for i, block_cache in enumerate(kv_cache):
|
||||
parts = block_cache[cache_branch]
|
||||
if parts["key"] is not entry.k[i]:
|
||||
entry.k[i].copy_(parts["key"])
|
||||
entry.v[i].copy_(parts["value"])
|
||||
parts["key"] = entry.k[i]
|
||||
parts["value"] = entry.v[i]
|
||||
# Graph buffers are scratch space; request caches must never alias them.
|
||||
entry.refresh_prefix(kv_cache)
|
||||
|
||||
if entry.graph is None:
|
||||
capture_error = None
|
||||
try:
|
||||
entry.capture(lambda: self.model._decode_graph_forward(entry))
|
||||
except Exception as exc:
|
||||
entry.failed = True
|
||||
entry.graph = None
|
||||
capture_error = str(exc)
|
||||
if capture_error is not None:
|
||||
self.entries[key] = None
|
||||
del entry
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph capture failed for key=%s: %s. "
|
||||
"Falling back to eager decode for this shape.",
|
||||
key,
|
||||
exc,
|
||||
capture_error,
|
||||
)
|
||||
return None
|
||||
logger.info("Captured Qwen-Image-2.1 decode graph for key=%s", key)
|
||||
|
||||
@@ -144,7 +144,7 @@ def get_qwen_image_21_pre_process_func(
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
if not raw_image: # None or empty list: pure text-to-image request
|
||||
if raw_image is None or (isinstance(raw_image, list) and len(raw_image) == 0):
|
||||
request.batch_compatibility_key = ("qwen_image_21", ())
|
||||
request.prompt = prompt
|
||||
return request
|
||||
@@ -744,6 +744,7 @@ class QwenImage21Pipeline(
|
||||
timesteps, num_inference_steps = retrieve_timesteps(
|
||||
self.scheduler,
|
||||
num_inference_steps,
|
||||
device=self.device,
|
||||
sigmas=sigmas,
|
||||
mu=mu,
|
||||
)
|
||||
|
||||
@@ -13,6 +13,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from diffusers.models.modeling_outputs import Transformer2DModelOutput
|
||||
from diffusers.models.normalization import RMSNorm
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -107,7 +108,10 @@ class QwenImage21TemporalTimesteps(nn.Module):
|
||||
self.time_factor = time_factor
|
||||
|
||||
half = timestep_dim // 2
|
||||
freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half)
|
||||
# Compute the fixed table on CPU like Diffusers, then honor the loader's device.
|
||||
freqs = torch.exp(
|
||||
-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device="cpu") / half
|
||||
).to(torch.get_default_device())
|
||||
self.register_buffer("freqs", freqs, persistent=False)
|
||||
|
||||
def forward(self, timestep: torch.Tensor) -> torch.Tensor:
|
||||
@@ -294,12 +298,15 @@ class QwenImage21Rope(nn.Module):
|
||||
self.theta = theta
|
||||
self.axes_dim = axes_dim
|
||||
|
||||
pos_index = torch.arange(8192)
|
||||
neg_index = torch.arange(1024).flip(0) * -1 - 1
|
||||
self.freqs = [
|
||||
torch.cat([self.rope_params(pos_index, dim, theta), self.rope_params(neg_index, dim, theta)], dim=0)
|
||||
for dim in axes_dim
|
||||
]
|
||||
# Match Diffusers' CPU initialization even when the loader sets a CUDA
|
||||
# default device; CPU and CUDA trigonometric kernels round differently.
|
||||
with torch.device("cpu"):
|
||||
pos_index = torch.arange(8192)
|
||||
neg_index = torch.arange(1024).flip(0) * -1 - 1
|
||||
self.freqs = [
|
||||
torch.cat([self.rope_params(pos_index, dim, theta), self.rope_params(neg_index, dim, theta)], dim=0)
|
||||
for dim in axes_dim
|
||||
]
|
||||
|
||||
def rope_params(self, index: torch.Tensor, dim: int, theta: int = 10000) -> torch.Tensor:
|
||||
freqs = torch.outer(index, 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)))
|
||||
@@ -374,9 +381,10 @@ class QwenImage21Attention(nn.Module):
|
||||
self.num_heads = self.to_qkv.num_heads
|
||||
self.num_kv_heads = self.to_qkv.num_kv_heads
|
||||
|
||||
# Per-head RMSNorm over the head dim; TP shards whole heads, so the norm is TP-safe.
|
||||
self.norm_q = nn.RMSNorm(dim_head, eps=eps)
|
||||
self.norm_k = nn.RMSNorm(dim_head, eps=eps)
|
||||
# Diffusers normalizes in FP32, then casts before the learned scale.
|
||||
# Keep that rounding order for BF16 Q/K; TP shards whole heads.
|
||||
self.norm_q = RMSNorm(dim_head, eps=eps)
|
||||
self.norm_k = RMSNorm(dim_head, eps=eps)
|
||||
|
||||
self.to_out = RowParallelLinear(
|
||||
heads * dim_head,
|
||||
@@ -718,7 +726,6 @@ class QwenImage21Transformer2DModel(CachedTransformer):
|
||||
causal_block: bool = True,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "transformer",
|
||||
enable_cuda_graph_decode: bool | None = None,
|
||||
cuda_graph_max_decode_graphs: int = 8,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -790,14 +797,11 @@ class QwenImage21Transformer2DModel(CachedTransformer):
|
||||
# Module boundary where _sp_plan shards the target tokens + their RoPE freqs.
|
||||
self.sequence_prepare = QwenImage21SequencePrepare(self.img_in, self.pos_embed)
|
||||
|
||||
# Opt-in CUDA Graph capture of the fixed-shape KV-cache decode steps.
|
||||
# Default off: ``None`` defers to ``od_config.enable_cuda_graph_decode``.
|
||||
if enable_cuda_graph_decode is None:
|
||||
enable_cuda_graph_decode = bool(getattr(od_config, "enable_cuda_graph_decode", False))
|
||||
self.enable_cuda_graph_decode = enable_cuda_graph_decode
|
||||
# Use decode graphs unless the caller requests eager execution.
|
||||
self.enable_cuda_graph_decode = not od_config.enforce_eager
|
||||
self._decode_graph_manager = (
|
||||
QwenImage21DecodeGraphManager(self, max_entries=cuda_graph_max_decode_graphs)
|
||||
if enable_cuda_graph_decode
|
||||
if self.enable_cuda_graph_decode
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -972,7 +976,9 @@ class QwenImage21Transformer2DModel(CachedTransformer):
|
||||
timestep=timestep,
|
||||
kv_cache=kv_cache,
|
||||
cache_branch=cache_branch,
|
||||
target_tokens=math.prod(layout[-1]),
|
||||
img_shapes=layout,
|
||||
img_mask=img_mask,
|
||||
encoder_hidden_states_mask=encoder_hidden_states_mask,
|
||||
)
|
||||
if graph_output is not None:
|
||||
if not return_dict:
|
||||
@@ -1039,6 +1045,9 @@ class QwenImage21Transformer2DModel(CachedTransformer):
|
||||
joint_key_valid = F.pad(joint_key_valid, (0, sp_padding), value=False)
|
||||
|
||||
has_padding = joint_key_valid is not None and not bool(joint_key_valid.all())
|
||||
if not has_padding:
|
||||
# Match Diffusers and use the same mask-free dispatch in eager and graph decode.
|
||||
joint_key_valid = None
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
cache_write_len: int | None = None
|
||||
if is_decode:
|
||||
@@ -1082,13 +1091,12 @@ class QwenImage21Transformer2DModel(CachedTransformer):
|
||||
and kv_cache is not None
|
||||
and self._decode_graph_manager is not None
|
||||
):
|
||||
# Move the freshly written prefix K/V into fixed-address buffers so
|
||||
# decode steps can replay a captured CUDA graph against them.
|
||||
# Allocate graph scratch buffers; the request keeps its own prefix K/V.
|
||||
self._decode_graph_manager.register_prefill(
|
||||
kv_cache=kv_cache,
|
||||
cache_branch=cache_branch,
|
||||
prefix_len=cache_write_len,
|
||||
target_tokens=target_freqs.shape[0],
|
||||
img_shapes=layout,
|
||||
target_freqs=target_freqs,
|
||||
joint_key_valid=joint_key_valid,
|
||||
dtype=hidden_states.dtype,
|
||||
@@ -1113,9 +1121,9 @@ class QwenImage21Transformer2DModel(CachedTransformer):
|
||||
"""The exact decode-step computation, captured into a CUDA graph.
|
||||
|
||||
Mirrors the eager decode path of ``forward``: target tokens only,
|
||||
prefix K/V read from the entry's static buffers, no attention metadata
|
||||
(the registration gate excludes padded masks), and an all-ones
|
||||
modulation mask. All inputs live in the entry's fixed-address buffers.
|
||||
prefix K/V read from the entry's static buffers, the same mask-free
|
||||
attention dispatch as eager decode, and an all-ones modulation mask.
|
||||
All inputs live in the entry's fixed-address buffers.
|
||||
"""
|
||||
hidden_states = self.img_in(entry.hidden)
|
||||
timestep = torch.cat([entry.timestep, entry.timestep.new_zeros(1)], dim=0)
|
||||
|
||||
Reference in new issue
Block a user