Optimize Qwen-Image 2.1 attention and decode graph reuse (#7790)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
1 parent
f930348e79
commit
ab4cf24b3e
24 files changed
+1147
-414
No files matched your search
@@ -9,19 +9,14 @@ default cache stores prefix K/V in freshly allocated tensors, so a captured
|
||||
graph would bind addresses that the next request (or a step-mode cache merge)
|
||||
no longer owns.
|
||||
|
||||
This module keeps the prefix K/V in persistent, fixed-address buffers. Prefill
|
||||
copies its K/V into them and re-points the caller's cache dict at the buffers;
|
||||
decode then replays a captured graph that reads those buffers plus static input
|
||||
buffers (target latents, timestep, RoPE frequencies). One graph is captured per
|
||||
``(branch, batch, prefix_len, target_tokens, dtype, device, backend)`` key and
|
||||
reused across requests: a new prefill with the same key rewrites the buffers in
|
||||
place, which a captured graph reads correctly because it binds addresses, not
|
||||
contents.
|
||||
Requests retain ownership of their prefix K/V. Decode refreshes separate
|
||||
fixed-address graph buffers when the active prefix tensors change. Graphs
|
||||
are keyed by branch, batch, prefix length, complete image layout,
|
||||
dtype, device and backend, so equal token counts with different RoPE layouts stay separate.
|
||||
|
||||
Memory note: only the *prefix* K/V are persistent. The decode attention still
|
||||
concatenates prefix K/V with the current step's target K/V inside the captured
|
||||
region, and that transient tensor is served by the graph's memory pool, so the
|
||||
persistent footprint is the same as the existing dict cache.
|
||||
Memory note: graph entries own an additional copy of the prefix K/V. Decode
|
||||
attention concatenates that prefix with target K/V inside the captured region;
|
||||
transient tensors use the graph memory pool.
|
||||
|
||||
Fallbacks (all logged, all eager): CUDA unavailable, sequence/ring/tensor
|
||||
parallelism, HSDP/offload hooks, torch.compile'd blocks, KV-cache quantization,
|
||||
@@ -32,6 +27,7 @@ capture failure.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -39,6 +35,8 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.models.qwen_image_21.qwen_image_21_transformer import (
|
||||
QwenImage21Transformer2DModel,
|
||||
@@ -67,8 +65,8 @@ class QwenImage21DecodeGraphEntry:
|
||||
|
||||
Buffers are created outside inference mode so replay-side ``copy_`` is
|
||||
legal regardless of the caller's grad context. ``graph`` is None until the
|
||||
first decode step captures it; ``failed`` disables capture permanently for
|
||||
this key after an exception.
|
||||
first decode step captures it. Failed entries are replaced by a ``None``
|
||||
marker in the manager so their buffers can be released.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -99,15 +97,30 @@ class QwenImage21DecodeGraphEntry:
|
||||
# inference_mode, and the captured body takes views of this tensor
|
||||
# outside it, which is forbidden for inference tensors.
|
||||
self.freqs = freqs.clone()
|
||||
# Decode needs no mask once padding is excluded (see the manager's
|
||||
# prefill gate), so the captured body attends without metadata.
|
||||
self.attn_metadata = None
|
||||
self.attn_metadata = None
|
||||
# Per-block cache dicts with the same shape as the legacy protocol.
|
||||
self.block_caches = [{branch: {"key": self.k[i], "value": self.v[i]}} for i in range(len(kv_shapes))]
|
||||
self.graph: torch.cuda.CUDAGraph | None = None
|
||||
self.output: torch.Tensor | None = None
|
||||
self.failed = False
|
||||
self.captures = 0
|
||||
self.prefix_sources: list[tuple[weakref.ReferenceType[torch.Tensor], int | None]] = []
|
||||
|
||||
def refresh_prefix(self, kv_cache: list[dict[str, dict[str, torch.Tensor]]]) -> None:
|
||||
sources = [block[self.branch][name] for block in kv_cache for name in ("key", "value")]
|
||||
versions = [None if tensor.is_inference() else tensor._version for tensor in sources]
|
||||
if len(sources) == len(self.prefix_sources) and all(
|
||||
previous() is tensor and version == current_version
|
||||
for (previous, version), tensor, current_version in zip(self.prefix_sources, sources, versions)
|
||||
):
|
||||
return
|
||||
# Prefill replaces inference-mode tensors; decode treats them as immutable.
|
||||
# Weak references avoid retaining completed requests or mistaking a reused
|
||||
# allocation for the same prefix. Normal tensors also track in-place writes.
|
||||
self.prefix_sources = []
|
||||
for i, block in enumerate(kv_cache):
|
||||
self.k[i].copy_(block[self.branch]["key"])
|
||||
self.v[i].copy_(block[self.branch]["value"])
|
||||
self.prefix_sources = [(weakref.ref(tensor), version) for tensor, version in zip(sources, versions)]
|
||||
|
||||
def capture(self, body) -> None:
|
||||
"""Warm up on a side stream, then capture one decode step.
|
||||
@@ -119,10 +132,12 @@ class QwenImage21DecodeGraphEntry:
|
||||
"""
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
|
||||
for _ in range(2):
|
||||
body()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
try:
|
||||
with torch.cuda.stream(side), torch.inference_mode(False), torch.no_grad():
|
||||
for _ in range(2):
|
||||
body()
|
||||
finally:
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
# The platform-wide pool is shared with every other captured path in
|
||||
@@ -142,7 +157,7 @@ class QwenImage21DecodeGraphManager:
|
||||
def __init__(self, model: QwenImage21Transformer2DModel, max_entries: int = 8):
|
||||
self.model = model
|
||||
self.max_entries = max_entries
|
||||
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry] = OrderedDict()
|
||||
self.entries: OrderedDict[tuple, QwenImage21DecodeGraphEntry | None] = OrderedDict()
|
||||
self._static_eligible: bool | None = None
|
||||
|
||||
# ── eligibility ──
|
||||
@@ -202,17 +217,12 @@ class QwenImage21DecodeGraphManager:
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
prefix_len: int,
|
||||
target_tokens: int,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
target_freqs: torch.Tensor,
|
||||
joint_key_valid: torch.Tensor | None,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Move a freshly prefilled prefix K/V into static buffers.
|
||||
|
||||
The caller's ``kv_cache`` dicts are re-pointed at the static buffers, so
|
||||
the existing decode path reads fixed addresses and a captured graph can
|
||||
be replayed for later steps. Prefill itself always runs eagerly.
|
||||
"""
|
||||
"""Prepare graph buffers without changing the request's prefix K/V."""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
@@ -239,44 +249,55 @@ class QwenImage21DecodeGraphManager:
|
||||
cache_branch,
|
||||
batch_size,
|
||||
prefix_len,
|
||||
target_tokens,
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
dtype,
|
||||
first["key"].device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
|
||||
entry = self.entries.get(key)
|
||||
if entry is None:
|
||||
if key not in self.entries:
|
||||
while len(self.entries) >= self.max_entries:
|
||||
evicted_key, _ = self.entries.popitem(last=False)
|
||||
evicted_key, evicted_entry = self.entries.popitem(last=False)
|
||||
del evicted_entry
|
||||
logger.debug("Evicting decode graph entry %s (max_entries=%d)", evicted_key, self.max_entries)
|
||||
entry = QwenImage21DecodeGraphEntry(
|
||||
branch=cache_branch,
|
||||
batch_size=batch_size,
|
||||
prefix_len=prefix_len,
|
||||
target_tokens=target_tokens,
|
||||
in_channels=self.model.in_channels,
|
||||
dtype=dtype,
|
||||
device=first["key"].device,
|
||||
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
|
||||
kv_dtype=first["key"].dtype,
|
||||
freqs=target_freqs,
|
||||
)
|
||||
self.entries[key] = entry
|
||||
allocation_error = None
|
||||
try:
|
||||
self.entries[key] = QwenImage21DecodeGraphEntry(
|
||||
branch=cache_branch,
|
||||
batch_size=batch_size,
|
||||
prefix_len=prefix_len,
|
||||
target_tokens=target_freqs.shape[0],
|
||||
in_channels=self.model.in_channels,
|
||||
dtype=dtype,
|
||||
device=first["key"].device,
|
||||
kv_shapes=[block_cache[cache_branch]["key"].shape for block_cache in kv_cache],
|
||||
kv_dtype=first["key"].dtype,
|
||||
freqs=target_freqs,
|
||||
)
|
||||
except torch.OutOfMemoryError as exc:
|
||||
allocation_error = str(exc)
|
||||
if allocation_error is not None:
|
||||
# Keep a bounded failure marker without retaining tensors or the exception traceback.
|
||||
self.entries[key] = None
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph allocation failed for key=%s: %s. Falling back to eager decode.",
|
||||
key,
|
||||
allocation_error,
|
||||
)
|
||||
return
|
||||
else:
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
for i, block_cache in enumerate(kv_cache):
|
||||
parts = block_cache[cache_branch]
|
||||
entry.k[i].copy_(parts["key"])
|
||||
entry.v[i].copy_(parts["value"])
|
||||
parts["key"] = entry.k[i]
|
||||
parts["value"] = entry.v[i]
|
||||
entry = self.entries[key]
|
||||
if entry is not None:
|
||||
entry.prefix_sources = []
|
||||
|
||||
logger.debug(
|
||||
"Registered decode graph entry key=%s (entries=%d, captured=%d)",
|
||||
key,
|
||||
len(self.entries),
|
||||
sum(1 for e in self.entries.values() if e.graph is not None),
|
||||
sum(1 for e in self.entries.values() if e is not None and e.graph is not None),
|
||||
)
|
||||
|
||||
# ── decode replay ──
|
||||
@@ -288,11 +309,20 @@ class QwenImage21DecodeGraphManager:
|
||||
timestep: torch.Tensor,
|
||||
kv_cache: list[dict[str, dict[str, torch.Tensor]]],
|
||||
cache_branch: str,
|
||||
target_tokens: int,
|
||||
img_shapes: list[tuple[int, int, int]],
|
||||
img_mask: torch.Tensor,
|
||||
encoder_hidden_states_mask: torch.Tensor | None,
|
||||
) -> torch.Tensor | None:
|
||||
"""Replay the captured decode step, or return None to fall back to eager."""
|
||||
if not self.eligible() or torch.compiler.is_compiling():
|
||||
return None
|
||||
if _dynamic_lora_wrappers_present(self.model):
|
||||
self.clear()
|
||||
return None
|
||||
if encoder_hidden_states_mask is not None:
|
||||
text_positions = ~img_mask[0, : encoder_hidden_states_mask.shape[1]]
|
||||
if not bool(encoder_hidden_states_mask[:, text_positions].all()):
|
||||
return None
|
||||
first = kv_cache[0][cache_branch]
|
||||
cached_key = first["key"]
|
||||
if cached_key.device.type != "cuda":
|
||||
@@ -304,40 +334,36 @@ class QwenImage21DecodeGraphManager:
|
||||
cache_branch,
|
||||
cached_key.shape[0],
|
||||
cached_key.shape[1],
|
||||
target_tokens,
|
||||
tuple(tuple(shape) for shape in img_shapes),
|
||||
hidden_states.dtype,
|
||||
cached_key.device.index,
|
||||
self._backend_name(),
|
||||
)
|
||||
entry = self.entries.get(key)
|
||||
if entry is None or entry.failed:
|
||||
if entry is None:
|
||||
return None
|
||||
self.entries.move_to_end(key)
|
||||
|
||||
entry.hidden.copy_(hidden_states[:, -target_tokens:])
|
||||
entry.hidden.copy_(hidden_states[:, -entry.target_tokens :])
|
||||
entry.timestep.copy_(timestep)
|
||||
# The caller's cache dicts normally already reference the entry's
|
||||
# buffers (prefill re-pointed them). A step-mode batched decode instead
|
||||
# hands over freshly merged tensors; copy those into the static buffers.
|
||||
for i, block_cache in enumerate(kv_cache):
|
||||
parts = block_cache[cache_branch]
|
||||
if parts["key"] is not entry.k[i]:
|
||||
entry.k[i].copy_(parts["key"])
|
||||
entry.v[i].copy_(parts["value"])
|
||||
parts["key"] = entry.k[i]
|
||||
parts["value"] = entry.v[i]
|
||||
# Graph buffers are scratch space; request caches must never alias them.
|
||||
entry.refresh_prefix(kv_cache)
|
||||
|
||||
if entry.graph is None:
|
||||
capture_error = None
|
||||
try:
|
||||
entry.capture(lambda: self.model._decode_graph_forward(entry))
|
||||
except Exception as exc:
|
||||
entry.failed = True
|
||||
entry.graph = None
|
||||
capture_error = str(exc)
|
||||
if capture_error is not None:
|
||||
self.entries[key] = None
|
||||
del entry
|
||||
current_omni_platform.empty_cache()
|
||||
logger.warning(
|
||||
"Qwen-Image-2.1 CUDA graph capture failed for key=%s: %s. "
|
||||
"Falling back to eager decode for this shape.",
|
||||
key,
|
||||
exc,
|
||||
capture_error,
|
||||
)
|
||||
return None
|
||||
logger.info("Captured Qwen-Image-2.1 decode graph for key=%s", key)
|
||||
|
||||
Reference in new issue
Block a user